Use rearranged sql credentials in flyway task (#712)

* Use rearranged sql credentials in flyway task

Let the flyway tasks use the sql credential files set up for BEAM
pipelines.

Credential files have been created for each environment in GCS
at gs://${project}-beam/cloudsql/admin_credential.enc. All
project editors have access to this file, including the Dataflow
control service account.

Alpha and crash use the 'nomulus-tools-key' in their own project to
decrypt the credential file.

Sandbox and production use the 'nomulus-tools-key' in
domain-registry-dev to decrypt the credential file.

Note that this setup is temporary. It will become obsolete once
we migrate to Cloud Secret Manager for secret storage.

.gitignore

@@ -79,6 +79,10 @@ nomulus.iml
 nomulus.ipr
 nomulus.iws
 
+# Auto-generated java classes by Intellij
+*/src/main/generated/
+*/src/test/generated_tests/
+
 # VScode
 .vscode
 

build.gradle

@@ -25,7 +25,7 @@ buildscript {
 
   dependencies {
     classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.0.1'
-    classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.6.1"
+    classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.6.1'
     classpath 'org.sonatype.aether:aether-api:1.13.1'
     classpath 'org.sonatype.aether:aether-impl:1.13.1'
   }
@@ -49,6 +49,7 @@ plugins {
   id 'com.diffplug.gradle.spotless' version '3.25.0'
 
   id 'jacoco'
+  id 'com.dorongold.task-tree' version '1.5'
 }
 
 wrapper {
@@ -443,6 +444,7 @@ task javaIncrementalFormatDryRun {
     println("${invokeJavaDiffFormatScript("show")}")
   }
 }
+tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)
 
 // Checks if modified lines in Java source files need reformatting.
 // Note that this task processes modified Java files in the entire repository.
@@ -452,8 +454,6 @@ task javaIncrementalFormatApply {
   }
 }
 
-tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)
-
 task javadoc(type: Javadoc) {
   source javadocSource
   classpath = files(javadocClasspath)
@@ -468,4 +468,16 @@ task javadoc(type: Javadoc) {
 
 tasks.build.dependsOn(tasks.javadoc)
 
+// Task for doing development on core Nomulus.
+// This fixes code formatting automatically as necessary, builds and tests the
+// core Nomulus codebase, and runs all presubmits.
+task coreDev {
+  dependsOn 'javaIncrementalFormatApply'
+  dependsOn 'javadoc'
+  dependsOn 'checkDependenciesDotGradle'
+  dependsOn 'checkLicense'
+  dependsOn ':core:check'
+  dependsOn 'assemble'
+}
+
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }

buildSrc/gradle/dependency-locks/testCompile.lockfile

@@ -61,12 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -61,12 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testRuntime.lockfile

@@ -61,12 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -61,12 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java

@@ -29,6 +29,7 @@ import google.registry.gradle.plugin.ProjectData.TaskData;
 import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Map;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -162,7 +163,7 @@ final class CoverPageGenerator {
             task.reports().entrySet().stream()
                 .collect(
                     toImmutableMap(
-                        entry -> entry.getKey(),
+                        Map.Entry::getKey,
                         entry ->
                             entry.getValue().files().isEmpty()
                                 ? ""

buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java

@@ -168,7 +168,7 @@ final class GcsPluginUtils {
             .collect(
                 toImmutableMap(
                     file -> rootDir.relativize(toNormalizedPath(file)),
-                    file -> toByteArraySupplier(file)));
+                    GcsPluginUtils::toByteArraySupplier));
 
     if (files.isEmpty()) {
       // The directory exists, but is empty. Return empty FilesWithEntryPoint

common/gradle/dependency-locks/testCompile.lockfile

@@ -20,10 +20,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -20,10 +20,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testRuntime.lockfile

@@ -21,10 +21,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -21,10 +21,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0

common/src/testing/java/google/registry/testing/truth/TextDiffSubject.java

@@ -16,6 +16,7 @@ package google.registry.testing.truth;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.truth.Truth.assertAbout;
+import static com.google.common.truth.Truth.assertWithMessage;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.github.difflib.DiffUtils;
@@ -31,6 +32,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.io.Resources;
 import com.google.common.truth.Fact;
 import com.google.common.truth.FailureMetadata;
+import com.google.common.truth.SimpleSubjectBuilder;
 import com.google.common.truth.Subject;
 import java.io.IOException;
 import java.net.URL;
@@ -68,6 +70,15 @@ public class TextDiffSubject extends Subject {
     this.actual = ImmutableList.copyOf(actual);
   }
 
+  protected TextDiffSubject(FailureMetadata metadata, URL actual) {
+    super(metadata, actual);
+    try {
+      this.actual = ImmutableList.copyOf(Resources.asCharSource(actual, UTF_8).readLines());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   public TextDiffSubject withDiffFormat(DiffFormat format) {
     this.diffFormat = format;
     return this;
@@ -100,6 +111,11 @@ public class TextDiffSubject extends Subject {
     return assertThat(Resources.asCharSource(resourceUrl, UTF_8).readLines());
   }
 
+  public static SimpleSubjectBuilder<TextDiffSubject, URL> assertWithMessageAboutUrlSource(
+      String format, Object... params) {
+    return assertWithMessage(format, params).about(urlFactory());
+  }
+
   private static final Subject.Factory<TextDiffSubject, ImmutableList<String>>
       TEXT_DIFF_SUBJECT_TEXT_FACTORY = TextDiffSubject::new;
 
@@ -107,6 +123,13 @@ public class TextDiffSubject extends Subject {
     return TEXT_DIFF_SUBJECT_TEXT_FACTORY;
   }
 
+  private static final Subject.Factory<TextDiffSubject, URL> TEXT_DIFF_SUBJECT_URL_FACTORY =
+      TextDiffSubject::new;
+
+  public static Subject.Factory<TextDiffSubject, URL> urlFactory() {
+    return TEXT_DIFF_SUBJECT_URL_FACTORY;
+  }
+
   static String generateUnifiedDiff(
       ImmutableList<String> expectedContent, ImmutableList<String> actualContent) {
     Patch<String> diff;

config/nom_build.py

@@ -49,10 +49,10 @@ PROPERTIES_HEADER = """\
 # This file defines properties used by the gradle build.  It must be kept in
 # sync with config/nom_build.py.
 #
-# To regenerate, run config/nom_build.py --generate-gradle-properties
+# To regenerate, run ./nom_build --generate-gradle-properties
 #
 # To view property descriptions (which are command line flags for
-# nom_build), run config/nom_build.py --help.
+# nom_build), run ./nom_build --help.
 #
 # DO NOT EDIT THIS FILE BY HAND
 org.gradle.jvmargs=-Xmx1024m
@@ -114,6 +114,11 @@ PROPERTIES = [
     Property('nomulus_version',
              'The version of nomulus to test against in a database '
              'integration test.'),
+    Property('dot_path',
+             'The path to "dot", part of the graphviz package that converts '
+             'a BEAM pipeline to image. Setting this property to empty string '
+             'will disable image generation.',
+             '/usr/bin/dot'),
 ]
 
 GRADLE_FLAGS = [

config/presubmits.py

@@ -22,7 +22,7 @@ import sys
 import re
 
 # We should never analyze any generated files
-UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/"}
+UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/"}
 # We can't rely on CI to have the Enum package installed so we do this instead.
 FORBIDDEN = 1
 REQUIRED = 2
@@ -77,9 +77,9 @@ PRESUBMITS = {
     PresubmitCheck(
         r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
         ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
-            ".git", "/build/", "/generated/", "node_modules/",
-            "JUnitBackports.java", "registrar_bin.", "registrar_dbg.",
-            "google-java-format-diff.py",
+            ".git", "/build/", "/generated/", "/generated_tests/",
+            "node_modules/", "JUnitBackports.java", "registrar_bin.",
+            "registrar_dbg.", "google-java-format-diff.py",
             "nomulus.golden.sql", "soyutils_usegoog.js"
         }, REQUIRED):
         "File did not include the license header.",

core/build.gradle

@@ -238,6 +238,7 @@ dependencies {
   compile deps['jline:jline']
   compile deps['joda-time:joda-time']
   compile deps['org.apache.avro:avro']
+  testCompile deps['org.apache.beam:beam-runners-core-construction-java']
   testCompile deps['org.apache.beam:beam-runners-direct-java']
   compile deps['org.apache.beam:beam-runners-google-cloud-dataflow-java']
   compile deps['org.apache.beam:beam-sdks-java-core']
@@ -256,6 +257,7 @@ dependencies {
   compile deps['org.bouncycastle:bcpg-jdk15on']
   testCompile deps['org.bouncycastle:bcpkix-jdk15on']
   compile deps['org.bouncycastle:bcprov-jdk15on']
+  testCompile deps['com.fasterxml.jackson.core:jackson-databind']
   runtime deps['org.glassfish.jaxb:jaxb-runtime']
   compile deps['org.hibernate:hibernate-core']
   compile deps['org.joda:joda-money']
@@ -312,6 +314,7 @@ dependencies {
   testCompile deps['org.junit.jupiter:junit-jupiter-api']
   testCompile deps['org.junit.jupiter:junit-jupiter-engine']
   testCompile deps['org.junit.jupiter:junit-jupiter-migrationsupport']
+  testCompile deps['org.junit.jupiter:junit-jupiter-params']
   testCompile deps['org.junit.platform:junit-platform-runner']
   testCompile deps['org.junit.platform:junit-platform-suite-api']
   testCompile deps['org.junit.vintage:junit-vintage-engine']
@@ -966,6 +969,49 @@ task buildToolImage(dependsOn: nomulus, type: Exec) {
   commandLine 'docker', 'build', '-t', 'nomulus-tool', '.'
 }
 
+task generateInitSqlPipelineGraph(type: Test) {
+  include "**/InitSqlPipelineGraphTest.*"
+  testNameIncludePatterns = ["**createPipeline_compareGraph"]
+  ignoreFailures = true
+}
+
+task updateInitSqlPipelineGraph(type: Copy) {
+  def graphRelativePath = 'google/registry/beam/initsql/'
+  from ("${projectDir}/build/resources/test/${graphRelativePath}") {
+    include 'pipeline_curr.dot'
+    rename 'curr', 'golden'
+  }
+  into "src/test/resources/${graphRelativePath}"
+
+  dependsOn generateInitSqlPipelineGraph
+
+  doLast {
+    if (com.google.common.base.Strings.isNullOrEmpty(project.dot_path)) {
+      getLogger().info('Property dot_path is null. Not creating image for pipeline graph.')
+    }
+    def dotPath = project.dot_path
+    if (!new File(dotPath).exists()) {
+      throw new RuntimeException(
+          """\
+              ${dotPath} not found. Make sure graphviz is installed
+              and the dot_path property is set correctly."""
+              .stripIndent())
+    }
+    def goldenGraph = "src/test/resources/${graphRelativePath}/pipeline_golden.dot"
+    def goldenImage = "src/test/resources/${graphRelativePath}/pipeline_golden.png"
+    def cmd = "${dotPath} -Tpng -o \"${goldenImage}\" \"${goldenGraph}\""
+    try {
+      rootProject.ext.execInBash(cmd, projectDir)
+    } catch (Throwable throwable) {
+      throw new RuntimeException(
+          """\
+              Failed to generate golden image with command ${cmd}
+              Error: ${throwable.getMessage()}
+          """)
+    }
+  }
+}
+
 // Build the devtool jar.
 createUberJar(
     'devtool',

core/gradle/dependency-locks/testCompile.lockfile

@@ -246,6 +246,7 @@ org.json:json:20160810
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -244,6 +244,7 @@ org.json:json:20160810
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2

core/gradle/dependency-locks/testRuntime.lockfile

@@ -249,6 +249,7 @@ org.json:json:20160810
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -249,6 +249,7 @@ org.json:json:20160810
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2

core/src/main/java/google/registry/backup/AppEngineEnvironment.java

@@ -22,20 +22,35 @@ import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 
 /**
- * Sets up a placeholder {@link Environment} on a non-AppEngine platform so that Datastore Entities
- * can be deserialized. See {@code DatastoreEntityExtension} in test source for more information.
+ * Sets up a fake {@link Environment} so that the following operations can be performed without the
+ * Datastore service:
+ *
+ * <ul>
+ *   <li>Create Objectify {@code Keys}.
+ *   <li>Instantiate Objectify objects.
+ *   <li>Convert Datastore {@code Entities} to their corresponding Objectify objects.
+ * </ul>
+ *
+ * <p>User has the option to specify their desired {@code appId} string, which forms part of an
+ * Objectify {@code Key} and is included in the equality check. This feature makes it easy to
+ * compare a migrated object in SQL with the original in Objectify.
+ *
+ * <p>Note that conversion from Objectify objects to Datastore {@code Entities} still requires the
+ * Datastore service.
  */
 public class AppEngineEnvironment implements Closeable {
 
-  private static final Environment PLACEHOLDER_ENV = createAppEngineEnvironment();
-
   private boolean isPlaceHolderNeeded;
 
   public AppEngineEnvironment() {
+    this("PlaceholderAppId");
+  }
+
+  public AppEngineEnvironment(String appId) {
     isPlaceHolderNeeded = ApiProxy.getCurrentEnvironment() == null;
     // isPlaceHolderNeeded may be true when we are invoked in a test with AppEngineRule.
     if (isPlaceHolderNeeded) {
-      ApiProxy.setEnvironmentForCurrentThread(PLACEHOLDER_ENV);
+      ApiProxy.setEnvironmentForCurrentThread(createAppEngineEnvironment(appId));
     }
   }
 
@@ -47,7 +62,7 @@ public class AppEngineEnvironment implements Closeable {
   }
 
   /** Returns a placeholder {@link Environment} that can return hardcoded AppId and Attributes. */
-  private static Environment createAppEngineEnvironment() {
+  private static Environment createAppEngineEnvironment(String appId) {
     return (Environment)
         Proxy.newProxyInstance(
             Environment.class.getClassLoader(),
@@ -55,7 +70,7 @@ public class AppEngineEnvironment implements Closeable {
             (Object proxy, Method method, Object[] args) -> {
               switch (method.getName()) {
                 case "getAppId":
-                  return "PlaceholderAppId";
+                  return appId;
                 case "getAttributes":
                   return ImmutableMap.<String, Object>of();
                 default:

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -169,10 +169,12 @@ public final class AsyncTaskEnqueuer {
         lock.getRelockDuration().isPresent(),
         "Lock with ID %s not configured for relock",
         lock.getRevisionId());
+    String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
     addTaskToQueueWithRetry(
         asyncActionsPushQueue,
         TaskOptions.Builder.withUrl(RelockDomainAction.PATH)
             .method(Method.POST)
+            .header("Host", backendHostname)
             .param(
                 RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
                 String.valueOf(lock.getRevisionId()))

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -532,7 +532,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
           .setKey(resourceKey)
-          .setLastUpdateTime(resource.getUpdateAutoTimestamp().getTimestamp())
+          .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(
                   params.get(PARAM_REQUESTING_CLIENT_ID), "Requesting client id not specified"))

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -319,13 +319,13 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
       HostResource host =
           checkNotNull(ofy().load().key(hostKey).now(), "Host to refresh doesn't exist");
       boolean isHostDeleted =
-          isDeleted(host, latestOf(now, host.getUpdateAutoTimestamp().getTimestamp()));
+          isDeleted(host, latestOf(now, host.getUpdateTimestamp().getTimestamp()));
       if (isHostDeleted) {
         logger.atInfo().log("Host %s is already deleted, not refreshing DNS.", hostKey);
       }
       return new AutoValue_RefreshDnsOnHostRenameAction_DnsRefreshRequest.Builder()
           .setHostKey(hostKey)
-          .setLastUpdateTime(host.getUpdateAutoTimestamp().getTimestamp())
+          .setLastUpdateTime(host.getUpdateTimestamp().getTimestamp())
           .setRequestedTime(
               DateTime.parse(
                   checkNotNull(params.get(PARAM_REQUESTED_TIME), "Requested time not specified")))

core/src/main/java/google/registry/beam/initsql/BeamJpaModule.java

@@ -19,31 +19,26 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
 
 import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
-import dagger.Binds;
 import dagger.Component;
 import dagger.Lazy;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.beam.initsql.BeamJpaModule.BindModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.JdbcJpaTm;
 import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
-import google.registry.util.Clock;
-import google.registry.util.Sleeper;
-import google.registry.util.SystemClock;
-import google.registry.util.SystemSleeper;
+import google.registry.util.UtilsModule;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.channels.Channels;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
-import javax.inject.Named;
+import javax.annotation.Nullable;
 import javax.inject.Singleton;
 import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.fs.ResourceId;
@@ -53,27 +48,31 @@ import org.apache.beam.sdk.io.fs.ResourceId;
  *
  * <p>This module is intended for use in BEAM pipelines, and uses a BEAM utility to access GCS like
  * a regular file system.
- *
- * <p>Note that {@link google.registry.config.RegistryConfig.ConfigModule} cannot be used here,
- * since many bindings, especially KMS-related ones, are different.
  */
-@Module(includes = {BindModule.class})
-class BeamJpaModule {
+@Module
+public class BeamJpaModule {
 
   private static final String GCS_SCHEME = "gs://";
 
-  private final String credentialFilePath;
+  @Nullable private final String credentialFilePath;
 
   /**
    * Constructs a new instance of {@link BeamJpaModule}.
    *
+   * <p>Note: it is an unfortunately necessary antipattern to check for the validity of
+   * credentialFilePath in {@link #provideCloudSqlAccessInfo} rather than in the constructor.
+   * Unfortunately, this is a restriction imposed upon us by Dagger. Specifically, because we use
+   * this in at least one 1 {@link google.registry.tools.RegistryTool} command(s), it must be
+   * instantiated in {@code google.registry.tools.RegistryToolComponent} for all possible commands;
+   * Dagger doesn't permit it to ever be null. For the vast majority of commands, it will never be
+   * used (so a null credential file path is fine in those cases).
+   *
    * @param credentialFilePath the path to a Cloud SQL credential file. This must refer to either a
    *     real encrypted file on GCS as returned by {@link
    *     BackupPaths#getCloudSQLCredentialFilePatterns} or an unencrypted file on local filesystem
    *     with credentials to a test database.
    */
-  BeamJpaModule(String credentialFilePath) {
-    checkArgument(!isNullOrEmpty(credentialFilePath), "Null or empty credentialFilePath");
+  public BeamJpaModule(@Nullable String credentialFilePath) {
     this.credentialFilePath = credentialFilePath;
   }
 
@@ -85,6 +84,7 @@ class BeamJpaModule {
   @Provides
   @Singleton
   SqlAccessInfo provideCloudSqlAccessInfo(Lazy<CloudSqlCredentialDecryptor> lazyDecryptor) {
+    checkArgument(!isNullOrEmpty(credentialFilePath), "Null or empty credentialFilePath");
     String line = readOnlyLineFromCredentialFile();
     if (isCloudSqlCredential()) {
       line = lazyDecryptor.get().decrypt(line);
@@ -114,13 +114,13 @@ class BeamJpaModule {
   }
 
   @Provides
-  @Config("cloudSqlJdbcUrl")
+  @Config("beamCloudSqlJdbcUrl")
   String provideJdbcUrl(SqlAccessInfo sqlAccessInfo) {
     return sqlAccessInfo.jdbcUrl();
   }
 
   @Provides
-  @Config("cloudSqlInstanceConnectionName")
+  @Config("beamCloudSqlInstanceConnectionName")
   String provideSqlInstanceName(SqlAccessInfo sqlAccessInfo) {
     return sqlAccessInfo
         .cloudSqlInstanceName()
@@ -128,58 +128,45 @@ class BeamJpaModule {
   }
 
   @Provides
-  @Config("cloudSqlUsername")
+  @Config("beamCloudSqlUsername")
   String provideSqlUsername(SqlAccessInfo sqlAccessInfo) {
     return sqlAccessInfo.user();
   }
 
   @Provides
-  @Config("cloudSqlPassword")
+  @Config("beamCloudSqlPassword")
   String provideSqlPassword(SqlAccessInfo sqlAccessInfo) {
     return sqlAccessInfo.password();
   }
 
   @Provides
-  @Config("cloudKmsProjectId")
+  @Config("beamCloudKmsProjectId")
   static String kmsProjectId() {
     return "domain-registry-dev";
   }
 
   @Provides
-  @Config("cloudKmsKeyRing")
+  @Config("beamCloudKmsKeyRing")
   static String keyRingName() {
     return "nomulus-tool-keyring";
   }
 
   @Provides
-  @Config("defaultCredentialOauthScopes")
-  static ImmutableList<String> defaultCredentialOauthScopes() {
-    return ImmutableList.of("https://www.googleapis.com/auth/cloud-platform");
-  }
-
-  @Provides
-  @Named("transientFailureRetries")
-  static int transientFailureRetries() {
-    return 12;
-  }
-
-  @Module
-  interface BindModule {
-
-    @Binds
-    Sleeper sleeper(SystemSleeper sleeper);
-
-    @Binds
-    Clock clock(SystemClock clock);
+  @Config("beamHibernateHikariMaximumPoolSize")
+  static int getBeamHibernateHikariMaximumPoolSize() {
+    // TODO(weiminyu): make this configurable. Should be equal to number of cores.
+    return 4;
   }
 
   @Singleton
   @Component(
       modules = {
+        ConfigModule.class,
         CredentialModule.class,
         BeamJpaModule.class,
         KmsModule.class,
-        PersistenceModule.class
+        PersistenceModule.class,
+        UtilsModule.class
       })
   public interface JpaTransactionManagerComponent {
     @SocketFactoryJpaTm

core/src/main/java/google/registry/beam/initsql/CloudSqlCredentialDecryptor.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.api.services.cloudkms.v1.model.DecryptRequest;
 import com.google.common.base.Strings;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.kms.KmsConnection;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
@@ -34,7 +35,7 @@ public class CloudSqlCredentialDecryptor {
   private final KmsConnection kmsConnection;
 
   @Inject
-  CloudSqlCredentialDecryptor(KmsConnection kmsConnection) {
+  CloudSqlCredentialDecryptor(@Config("beamKmsConnection") KmsConnection kmsConnection) {
     this.kmsConnection = kmsConnection;
   }
 

core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java

@@ -0,0 +1,75 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.appengine.api.datastore.Entity;
+import java.util.Objects;
+
+/** Helper for manipulating {@code DomainBase} when migrating from Datastore to SQL database */
+final class DomainBaseUtil {
+
+  private DomainBaseUtil() {}
+
+  /**
+   * Removes {@link google.registry.model.billing.BillingEvent.Recurring}, {@link
+   * google.registry.model.poll.PollMessage PollMessages} and {@link
+   * google.registry.model.host.HostResource name servers} from a Datastore {@link Entity} that
+   * represents an Ofy {@link google.registry.model.domain.DomainBase}. This breaks the cycle of
+   * foreign key constraints between these entity kinds, allowing {@code DomainBases} to be inserted
+   * into the SQL database. See {@link InitSqlPipeline} for a use case, where the full {@code
+   * DomainBases} are written again during the last stage of the pipeline.
+   *
+   * <p>The returned object may be in bad state. Specifically, {@link
+   * google.registry.model.eppcommon.StatusValue#INACTIVE} is not added after name servers are
+   * removed. This only impacts tests.
+   *
+   * <p>This operation is performed on an Datastore {@link Entity} instead of Ofy Java object
+   * because Objectify requires access to a Datastore service when converting an Ofy object to a
+   * Datastore {@code Entity}. If we insist on working with Objectify objects, we face a few
+   * unsatisfactory options:
+   *
+   * <ul>
+   *   <li>Connect to our production Datastore, which incurs unnecessary security and code health
+   *       risk.
+   *   <li>Connect to a separate real Datastore instance, which is a waster and overkill.
+   *   <li>Use an in-memory test Datastore, which is a project health risk in that the test
+   *       Datastore would be added to Nomulus' production binary unless we create a separate
+   *       project for this pipeline.
+   * </ul>
+   *
+   * <p>Given our use case, operating on Datastore entities is the best option.
+   *
+   * @throws IllegalArgumentException if input does not represent a DomainBase
+   */
+  static Entity removeBillingAndPollAndHosts(Entity domainBase) {
+    checkNotNull(domainBase, "domainBase");
+    checkArgument(
+        Objects.equals(domainBase.getKind(), "DomainBase"),
+        "Expecting DomainBase, got %s",
+        domainBase.getKind());
+    Entity clone = domainBase.clone();
+    clone.removeProperty("autorenewBillingEvent");
+    clone.removeProperty("autorenewPollMessage");
+    clone.removeProperty("deletePollMessage");
+    clone.removeProperty("nsHosts");
+    domainBase.getProperties().keySet().stream()
+        .filter(s -> s.startsWith("transferData."))
+        .forEach(s -> clone.removeProperty(s));
+    return clone;
+  }
+}

core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java

@@ -0,0 +1,260 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.AppEngineEnvironment;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.BeamJpaModule.JpaTransactionManagerComponent;
+import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys;
+import google.registry.beam.initsql.Transforms.SerializableSupplier;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registry.Registry;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.Wait;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+
+/**
+ * A BEAM pipeline that populates a SQL database with data from a Datastore backup.
+ *
+ * <p>This pipeline migrates EPP resources and related entities that cross-reference each other. To
+ * avoid violating foreign key constraints, writes to SQL are ordered by entity kinds. In addition,
+ * the {@link DomainBase} kind is written twice (see details below). The write order is presented
+ * below. Although some kinds can be written concurrently, e.g. {@code ContactResource} and {@code
+ * RegistrarContact}, we do not expect any performance benefit since the limiting resource is the
+ * number of JDBC connections. Google internal users may refer to <a
+ * href="http://go/registry-r3-init-sql">the design doc</a> for more information.
+ *
+ * <ol>
+ *   <li>{@link Registry}: Assumes that {@code PremiumList} and {@code ReservedList} have been set
+ *       up in the SQL database.
+ *   <li>{@link Registrar}: Logically depends on {@code Registry}, Foreign key not modeled yet.
+ *   <li>{@link ContactResource}: references {@code Registrar}
+ *   <li>{@link RegistrarContact}: references {@code Registrar}.
+ *   <li>Cleansed {@link DomainBase}: with references to {@code BillingEvent}, {@code Recurring},
+ *       {@code Cancellation} and {@code HostResource} removed, still references {@code Registrar}
+ *       and {@code ContactResource}. The removal breaks circular Foreign Key references.
+ *   <li>{@link HostResource}: references {@code DomainBase}.
+ *   <li>{@link HistoryEntry}: maps to one of three SQL entity types and may reference {@code
+ *       Registrar}, {@code ContactResource}, {@code HostResource}, and {@code DomainBase}.
+ *   <li>{@link AllocationToken}: references {@code HistoryEntry}.
+ *   <li>{@link BillingEvent.Recurring}: references {@code Registrar}, {@code DomainBase} and {@code
+ *       HistoryEntry}.
+ *   <li>{@link BillingEvent.OneTime}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       BillingEvent.Recurring}, {@code HistoryEntry} and {@code AllocationToken}.
+ *   <li>{@link BillingEvent.Modification}: SQL model TBD. Will reference {@code Registrar}, {@code
+ *       DomainBase} and {@code BillingEvent.OneTime}.
+ *   <li>{@link BillingEvent.Cancellation}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       BillingEvent.Recurring}, {@code BillingEvent.OneTime}, and {@code HistoryEntry}.
+ *   <li>{@link PollMessage}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       ContactResource}, {@code HostResource}, and {@code HistoryEntry}.
+ *   <li>{@link DomainBase}, original copy from Datastore.
+ * </ol>
+ */
+public class InitSqlPipeline implements Serializable {
+
+  /**
+   * Datastore kinds to be written to the SQL database before the cleansed version of {@link
+   * DomainBase}.
+   */
+  // TODO(weiminyu): include Registry.class when it is modeled in JPA.
+  private static final ImmutableList<Class<?>> PHASE_ONE_ORDERED =
+      ImmutableList.of(Registrar.class, ContactResource.class);
+
+  /**
+   * Datastore kinds to be written to the SQL database after the cleansed version of {@link
+   * DomainBase}.
+   *
+   * <p>The following entities are missing from the list:
+   *
+   * <ul>
+   *   <li>Those not modeled in JPA yet, e.g., {@code BillingEvent.Modification}.
+   *   <li>Those waiting for sanitation, e.g., {@code HistoryEntry}, which would have duplicate keys
+   *       after converting to SQL model.
+   *   <li>Those that have foreign key constraints on the above.
+   * </ul>
+   */
+  // TODO(weiminyu): add more entities when available.
+  private static final ImmutableList<Class<?>> PHASE_TWO_ORDERED =
+      ImmutableList.of(HostResource.class);
+
+  private final InitSqlPipelineOptions options;
+
+  private final Pipeline pipeline;
+
+  private final SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager>
+      jpaGetter;
+
+  InitSqlPipeline(InitSqlPipelineOptions options) {
+    this.options = options;
+    pipeline = Pipeline.create(options);
+    jpaGetter = JpaTransactionManagerComponent::cloudSqlJpaTransactionManager;
+  }
+
+  @VisibleForTesting
+  InitSqlPipeline(InitSqlPipelineOptions options, Pipeline pipeline) {
+    this.options = options;
+    this.pipeline = pipeline;
+    jpaGetter = JpaTransactionManagerComponent::localDbJpaTransactionManager;
+  }
+
+  public PipelineResult run() {
+    setupPipeline();
+    return pipeline.run();
+  }
+
+  @VisibleForTesting
+  void setupPipeline() {
+    PCollectionTuple datastoreSnapshot =
+        pipeline.apply(
+            "Load Datastore snapshot",
+            Transforms.loadDatastoreSnapshot(
+                options.getDatastoreExportDir(),
+                options.getCommitLogDir(),
+                DateTime.parse(options.getCommitLogStartTimestamp()),
+                DateTime.parse(options.getCommitLogEndTimestamp()),
+                ImmutableSet.<String>builder()
+                    .add("DomainBase")
+                    .addAll(toKindStrings(PHASE_ONE_ORDERED))
+                    .addAll(toKindStrings(PHASE_TWO_ORDERED))
+                    .build()));
+
+    // Set up the pipeline to write entity kinds from PHASE_ONE_ORDERED to SQL. Return a object
+    // that signals the completion of the phase.
+    PCollection<Void> blocker =
+        scheduleOnePhaseWrites(datastoreSnapshot, PHASE_ONE_ORDERED, Optional.empty(), null);
+    blocker =
+        writeToSql(
+            "DomainBase without circular foreign keys",
+            removeDomainBaseForeignKeys(datastoreSnapshot)
+                .apply("Wait on phase one", Wait.on(blocker)));
+    // Set up the pipeline to write entity kinds from PHASE_TWO_ORDERED to SQL. This phase won't
+    // start until all cleansed DomainBases have been written (started by line above).
+    scheduleOnePhaseWrites(
+        datastoreSnapshot, PHASE_TWO_ORDERED, Optional.of(blocker), "DomainBaseNoFkeys");
+  }
+
+  private PCollection<VersionedEntity> removeDomainBaseForeignKeys(
+      PCollectionTuple datastoreSnapshot) {
+    PCollection<VersionedEntity> domainBases =
+        datastoreSnapshot.get(Transforms.createTagForKind("DomainBase"));
+    return domainBases.apply(
+        "Remove circular foreign keys from DomainBase",
+        ParDo.of(new RemoveDomainBaseForeignKeys()));
+  }
+
+  /**
+   * Sets up the pipeline to write entities in {@code entityClasses} to SQL. Entities are written
+   * one kind at a time based on each kind's position in {@code entityClasses}. Concurrency exists
+   * within each kind.
+   *
+   * @param datastoreSnapshot the Datastore snapshot of all data to be migrated to SQL
+   * @param entityClasses the entity types in write order
+   * @param blockingPCollection the pipeline stage that blocks this phase
+   * @param blockingTag description of the stage (if exists) that blocks this phase. Needed for
+   *     generating unique transform ids
+   * @return the output {@code PCollection} from the writing of the last entity kind. Other parts of
+   *     the pipeline can {@link Wait} on this object
+   */
+  private PCollection<Void> scheduleOnePhaseWrites(
+      PCollectionTuple datastoreSnapshot,
+      Collection<Class<?>> entityClasses,
+      Optional<PCollection<Void>> blockingPCollection,
+      String blockingTag) {
+    checkArgument(!entityClasses.isEmpty(), "Each phase must have at least one kind.");
+    ImmutableList<TupleTag<VersionedEntity>> tags =
+        toKindStrings(entityClasses).stream()
+            .map(Transforms::createTagForKind)
+            .collect(ImmutableList.toImmutableList());
+
+    PCollection<Void> prev = blockingPCollection.orElse(null);
+    String prevTag = blockingTag;
+    for (TupleTag<VersionedEntity> tag : tags) {
+      PCollection<VersionedEntity> curr = datastoreSnapshot.get(tag);
+      if (prev != null) {
+        curr = curr.apply("Wait on " + prevTag, Wait.on(prev));
+      }
+      prev = writeToSql(tag.getId(), curr);
+      prevTag = tag.getId();
+    }
+    return prev;
+  }
+
+  private PCollection<Void> writeToSql(String transformId, PCollection<VersionedEntity> data) {
+    String credentialFileUrl =
+        options.getSqlCredentialUrlOverride() != null
+            ? options.getSqlCredentialUrlOverride()
+            : BackupPaths.getCloudSQLCredentialFilePatterns(options.getEnvironment()).get(0);
+
+    return data.apply(
+        "Write to sql: " + transformId,
+        Transforms.writeToSql(
+            transformId,
+            options.getMaxConcurrentSqlWriters(),
+            options.getSqlWriteBatchSize(),
+            new JpaSupplierFactory(credentialFileUrl, jpaGetter)));
+  }
+
+  private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
+    try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+      return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
+    }
+  }
+
+  static class JpaSupplierFactory implements SerializableSupplier<JpaTransactionManager> {
+    private static final long serialVersionUID = 1L;
+
+    private String credentialFileUrl;
+    private SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager> jpaGetter;
+
+    JpaSupplierFactory(
+        String credentialFileUrl,
+        SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager> jpaGetter) {
+      this.credentialFileUrl = credentialFileUrl;
+      this.jpaGetter = jpaGetter;
+    }
+
+    @Override
+    public JpaTransactionManager get() {
+      return jpaGetter.apply(
+          DaggerBeamJpaModule_JpaTransactionManagerComponent.builder()
+              .beamJpaModule(new BeamJpaModule(credentialFileUrl))
+              .build());
+    }
+  }
+}

core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java

@@ -0,0 +1,84 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Validation;
+
+/** Pipeline options for {@link InitSqlPipeline} */
+public interface InitSqlPipelineOptions extends GcpOptions {
+
+  @Description(
+      "Overrides the URL to the SQL credential file. " + "Required if environment is not provided.")
+  @Nullable
+  String getSqlCredentialUrlOverride();
+
+  void setSqlCredentialUrlOverride(String credentialUrlOverride);
+
+  @Description("The root directory of the export to load.")
+  String getDatastoreExportDir();
+
+  void setDatastoreExportDir(String datastoreExportDir);
+
+  @Description("The directory that contains all CommitLog files.")
+  String getCommitLogDir();
+
+  void setCommitLogDir(String commitLogDir);
+
+  @Description("The earliest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getCommitLogStartTimestamp();
+
+  void setCommitLogStartTimestamp(String commitLogStartTimestamp);
+
+  @Description("The latest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getCommitLogEndTimestamp();
+
+  void setCommitLogEndTimestamp(String commitLogEndTimestamp);
+
+  @Description(
+      "The deployed environment, alpha, crash, sandbox, or production. "
+          + "Not required only if sqlCredentialUrlOverride is provided.")
+  @Nullable
+  String getEnvironment();
+
+  void setEnvironment(String environment);
+
+  @Description(
+      "The maximum JDBC connection pool size on a VM. "
+          + "This value should be equal to or greater than the number of cores on the VM.")
+  @Default.Integer(4)
+  int getJdbcMaxPoolSize();
+
+  void setJdbcMaxPoolSize(int jdbcMaxPoolSize);
+
+  @Description(
+      "A hint to the pipeline runner of the maximum number of concurrent SQL writers to create. "
+          + "Note that multiple writers may run on the same VM and share the connection pool.")
+  @Default.Integer(4)
+  int getMaxConcurrentSqlWriters();
+
+  void setMaxConcurrentSqlWriters(int maxConcurrentSqlWriters);
+
+  @Description("The number of entities to be written to the SQL database in one transaction.")
+  @Default.Integer(20)
+  int getSqlWriteBatchSize();
+
+  void setSqlWriteBatchSize(int sqlWriteBatchSize);
+}

core/src/main/java/google/registry/beam/initsql/README.md

@@ -1,3 +1,17 @@
 ## Summary
 
-This package contains a BEAM pipeline that populates a Cloud SQL database from a Datastore backup.
+This package contains a BEAM pipeline that populates a Cloud SQL database from a
+Datastore backup. The pipeline uses an unsynchronized Datastore export and
+overlapping CommitLogs generated by the Nomulus server to recreate a consistent
+Datastore snapshot, and writes the data to a Cloud SQL instance.
+
+## Pipeline Visualization
+
+The golden flow graph of the InitSqlPipeline is saved both as a text-base
+[DOT file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.dot)
+and a
+[.png file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.png).
+A test compares the flow graph of the current pipeline with the golden graph,
+and will fail if changes are detected. When this happens, run the Gradle task
+':core:updateInitSqlPipelineGraph' to update the golden files and review the
+changes.

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -16,18 +16,43 @@ package google.registry.beam.initsql;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Throwables.throwIfUnchecked;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
+import static google.registry.persistence.JpaRetries.isFailedTxnRetriable;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.setJpaTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static java.util.Comparator.comparing;
+import static org.apache.beam.sdk.values.TypeDescriptors.integers;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
 
 import avro.shaded.com.google.common.collect.Iterators;
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
+import google.registry.backup.AppEngineEnvironment;
 import google.registry.backup.CommitLogImports;
 import google.registry.backup.VersionedEntity;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.ofy.Ofy;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.tools.LevelDbLogReader;
+import google.registry.util.SystemSleeper;
+import java.io.Serializable;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Supplier;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.Compression;
 import org.apache.beam.sdk.io.FileIO;
@@ -36,12 +61,23 @@ import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.GroupIntoBatches;
+import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ProcessFunction;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.TypeDescriptor;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 
 /**
  * {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export
@@ -57,6 +93,110 @@ public final class Transforms {
    */
   @VisibleForTesting static final long EXPORT_ENTITY_TIME_STAMP = START_OF_TIME.getMillis();
 
+  /**
+   * Returns a {@link TupleTag} that can be used to retrieve entities of the given {@code kind} from
+   * the Datastore snapshot returned by {@link #loadDatastoreSnapshot}.
+   */
+  public static TupleTag<VersionedEntity> createTagForKind(String kind) {
+    // When used with PCollectionTuple the result must retain generic type information.
+    // Both the Generic param and the empty bracket below are important.
+    return new TupleTag<VersionedEntity>(Transforms.class.getSimpleName() + ":" + kind) {};
+  }
+
+  /**
+   * Composite {@link PTransform transform} that loads the Datastore snapshot at {@code
+   * commitLogToTime} for caller specified {@code kinds}.
+   *
+   * <p>Caller must provide the location of a Datastore export that started AFTER {@code
+   * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory
+   * of all CommitLog files.
+   *
+   * <p>Selection of {@code commitLogFromTime} and {@code commitLogToTime} should follow the
+   * guidelines below to ensure that all incremental changes concurrent with the export are covered:
+   *
+   * <ul>
+   *   <li>Two or more CommitLogs should exist between {@code commitLogFromTime} and the starting
+   *       time of the Datastore export. This ensures that the earlier CommitLog file was complete
+   *       before the export started.
+   *   <li>Two or more CommitLogs should exit between the export completion time and {@code
+   *       commitLogToTime}.
+   * </ul>
+   *
+   * <p>The output from the returned transform is a {@link PCollectionTuple} consisting of {@link
+   * VersionedEntity VersionedEntities} grouped into {@link PCollection PCollections} by {@code
+   * kind}.
+   */
+  public static PTransform<PBegin, PCollectionTuple> loadDatastoreSnapshot(
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<String> kinds) {
+    checkArgument(kinds != null && !kinds.isEmpty(), "At least one kind is expected.");
+
+    // Create tags to collect entities by kind in final step.
+    final ImmutableMap<String, TupleTag<VersionedEntity>> outputTags =
+        kinds.stream()
+            .collect(ImmutableMap.toImmutableMap(kind -> kind, Transforms::createTagForKind));
+    // Arbitrarily select one tag as mainOutTag and put the remaining ones in a TupleTagList.
+    // This separation is required by ParDo's config API.
+    Iterator<TupleTag<VersionedEntity>> tagsIt = outputTags.values().iterator();
+    final TupleTag<VersionedEntity> mainOutputTag = tagsIt.next();
+    final TupleTagList additionalTags = TupleTagList.of(ImmutableList.copyOf(tagsIt));
+
+    return new PTransform<PBegin, PCollectionTuple>() {
+      @Override
+      public PCollectionTuple expand(PBegin input) {
+        PCollection<VersionedEntity> exportedEntities =
+            input
+                .apply("Get export file patterns", getDatastoreExportFilePatterns(exportDir, kinds))
+                .apply("Find export files", getFilesByPatterns())
+                .apply("Load export data", loadExportDataFromFiles());
+        PCollection<VersionedEntity> commitLogEntities =
+            input
+                .apply("Get commitlog file patterns", getCommitLogFilePatterns(commitLogDir))
+                .apply("Find commitlog files", getFilesByPatterns())
+                .apply(
+                    "Filter commitLog by time",
+                    filterCommitLogsByTime(commitLogFromTime, commitLogToTime))
+                .apply("Load commitlog data", loadCommitLogsFromFiles(kinds));
+        return PCollectionList.of(exportedEntities)
+            .and(commitLogEntities)
+            .apply("Merge exports and CommitLogs", Flatten.pCollections())
+            .apply(
+                "Key entities by Datastore Keys",
+                // Converting to KV<String, VE> instead of KV<Key, VE> b/c default coder for Key
+                // (SerializableCoder) is not deterministic and cannot be used with GroupBy.
+                MapElements.into(kvs(strings(), TypeDescriptor.of(VersionedEntity.class)))
+                    .via((VersionedEntity e) -> KV.of(e.key().toString(), e)))
+            .apply("Gather entities by key", GroupByKey.create())
+            .apply(
+                "Output latest version per entity",
+                ParDo.of(
+                        new DoFn<KV<String, Iterable<VersionedEntity>>, VersionedEntity>() {
+                          @ProcessElement
+                          public void processElement(
+                              @Element KV<String, Iterable<VersionedEntity>> kv,
+                              MultiOutputReceiver out) {
+                            Optional<VersionedEntity> latest =
+                                Streams.stream(kv.getValue())
+                                    .sorted(comparing(VersionedEntity::commitTimeMills).reversed())
+                                    .findFirst();
+                            // Throw to abort (after default retries). Investigate, fix, and rerun.
+                            checkState(
+                                latest.isPresent(), "Unexpected key with no data", kv.getKey());
+                            if (latest.get().isDelete()) {
+                              return;
+                            }
+                            String kind = latest.get().getEntity().get().getKind();
+                            out.get(outputTags.get(kind)).output(latest.get());
+                          }
+                        })
+                    .withOutputTags(mainOutputTag, additionalTags));
+      }
+    };
+  }
+
   /**
    * Returns a {@link PTransform transform} that can generate a collection of patterns that match
    * all Datastore CommitLog files.
@@ -87,7 +227,7 @@ public final class Transforms {
     return new PTransform<PCollection<String>, PCollection<Metadata>>() {
       @Override
       public PCollection<Metadata> expand(PCollection<String> input) {
-        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW));
       }
     };
   }
@@ -96,7 +236,7 @@ public final class Transforms {
    * Returns CommitLog files with timestamps between {@code fromTime} (inclusive) and {@code
    * endTime} (exclusive).
    */
-  public static PTransform<PCollection<? extends String>, PCollection<String>>
+  public static PTransform<PCollection<? extends Metadata>, PCollection<Metadata>>
       filterCommitLogsByTime(DateTime fromTime, DateTime toTime) {
     return ParDo.of(new FilterCommitLogFileByTime(fromTime, toTime));
   }
@@ -114,11 +254,51 @@ public final class Transforms {
 
   /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
   public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
-      loadCommitLogsFromFiles() {
+      loadCommitLogsFromFiles(Set<String> kinds) {
     return processFiles(
-        new BackupFileReader(file -> CommitLogImports.loadEntities(file.open()).iterator()));
+        new BackupFileReader(
+            file ->
+                CommitLogImports.loadEntities(file.open()).stream()
+                    .filter(e -> kinds.contains(e.key().getKind()))
+                    .iterator()));
   }
 
+  /**
+   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
+   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
+   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   *
+   * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
+   * job, we will not add features unless proven necessary.
+   *
+   * @param transformId a unique ID for an instance of the returned transform
+   * @param maxWriters the max number of concurrent writes to SQL, which also determines the max
+   *     number of connection pools created
+   * @param batchSize the number of entities to write in each operation
+   * @param jpaSupplier supplier of a {@link JpaTransactionManager}
+   */
+  public static PTransform<PCollection<VersionedEntity>, PCollection<Void>> writeToSql(
+      String transformId,
+      int maxWriters,
+      int batchSize,
+      SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+    return new PTransform<PCollection<VersionedEntity>, PCollection<Void>>() {
+      @Override
+      public PCollection<Void> expand(PCollection<VersionedEntity> input) {
+        return input
+            .apply(
+                "Shard data for " + transformId,
+                MapElements.into(kvs(integers(), TypeDescriptor.of(VersionedEntity.class)))
+                    .via(ve -> KV.of(ThreadLocalRandom.current().nextInt(maxWriters), ve)))
+            .apply("Batch output by shard " + transformId, GroupIntoBatches.ofSize(batchSize))
+            .apply("Write in batch for " + transformId, ParDo.of(new SqlBatchWriter(jpaSupplier)));
+      }
+    };
+  }
+
+  /** Interface for serializable {@link Supplier suppliers}. */
+  public interface SerializableSupplier<T> extends Supplier<T>, Serializable {}
+
   /**
    * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
    * given {@link Iterable}.
@@ -139,12 +319,11 @@ public final class Transforms {
         return input
             .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
             .apply(transformer.getClass().getSimpleName(), ParDo.of(transformer));
-        // TODO(weiminyu): reshuffle to enable dynamic work rebalance per beam dev guide
       }
     };
   }
 
-  private static class FilterCommitLogFileByTime extends DoFn<String, String> {
+  private static class FilterCommitLogFileByTime extends DoFn<Metadata, Metadata> {
     private final DateTime fromTime;
     private final DateTime toTime;
 
@@ -161,10 +340,10 @@ public final class Transforms {
     }
 
     @ProcessElement
-    public void processElement(@Element String fileName, OutputReceiver<String> out) {
-      DateTime timestamp = getCommitLogTimestamp(fileName);
+    public void processElement(@Element Metadata fileMeta, OutputReceiver<Metadata> out) {
+      DateTime timestamp = getCommitLogTimestamp(fileMeta.resourceId().toString());
       if (isBeforeOrAt(fromTime, timestamp) && timestamp.isBefore(toTime)) {
-        out.output(fileName);
+        out.output(fileMeta);
       }
     }
   }
@@ -197,4 +376,116 @@ public final class Transforms {
       }
     }
   }
+
+  /**
+   * Writes a batch of entities to a SQL database.
+   *
+   * <p>Note that an arbitrary number of instances of this class may be created and freed in
+   * arbitrary order in a single JVM. Due to the tech debt that forced us to use a static variable
+   * to hold the {@code JpaTransactionManager} instance, we must ensure that JpaTransactionManager
+   * is not changed or torn down while being used by some instance.
+   */
+  private static class SqlBatchWriter extends DoFn<KV<Integer, Iterable<VersionedEntity>>, Void> {
+
+    private static int instanceCount = 0;
+    private static JpaTransactionManager originalJpa;
+
+    private final SerializableSupplier<JpaTransactionManager> jpaSupplier;
+
+    private transient Ofy ofy;
+    private transient SystemSleeper sleeper;
+
+    SqlBatchWriter(SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+      this.jpaSupplier = jpaSupplier;
+    }
+
+    @Setup
+    public void setup() {
+      sleeper = new SystemSleeper();
+
+      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+        ObjectifyService.initOfy();
+        ofy = ObjectifyService.ofy();
+      }
+
+      synchronized (SqlBatchWriter.class) {
+        if (instanceCount == 0) {
+          originalJpa = jpaTm();
+          setJpaTm(jpaSupplier);
+        }
+        instanceCount++;
+      }
+    }
+
+    @Teardown
+    public void teardown() {
+      synchronized (SqlBatchWriter.class) {
+        instanceCount--;
+        if (instanceCount == 0) {
+          jpaTm().teardown();
+          setJpaTm(() -> originalJpa);
+        }
+      }
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<Integer, Iterable<VersionedEntity>> kv) {
+      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+        ImmutableList<Object> ofyEntities =
+            Streams.stream(kv.getValue())
+                .map(VersionedEntity::getEntity)
+                .map(Optional::get)
+                .map(ofy::toPojo)
+                .collect(ImmutableList.toImmutableList());
+        retry(() -> jpaTm().transact(() -> jpaTm().saveNewOrUpdateAll(ofyEntities)));
+      }
+    }
+
+    // TODO(b/160632289): Enhance Retrier and use it here.
+    private void retry(Runnable runnable) {
+      int maxAttempts = 5;
+      int initialDelayMillis = 100;
+      double jitterRatio = 0.2;
+
+      for (int attempt = 0; attempt < maxAttempts; attempt++) {
+        try {
+          runnable.run();
+          return;
+        } catch (Throwable throwable) {
+          if (!isFailedTxnRetriable(throwable)) {
+            throwIfUnchecked(throwable);
+            throw new RuntimeException(throwable);
+          }
+          int sleepMillis = (1 << attempt) * initialDelayMillis;
+          int jitter =
+              ThreadLocalRandom.current().nextInt((int) (sleepMillis * jitterRatio))
+                  - (int) (sleepMillis * jitterRatio / 2);
+          sleeper.sleepUninterruptibly(Duration.millis(sleepMillis + jitter));
+        }
+      }
+    }
+  }
+
+  /**
+   * Removes BillingEvents, {@link google.registry.model.poll.PollMessage PollMessages} and {@link
+   * google.registry.model.host.HostResource} from a {@link DomainBase}. These are circular foreign
+   * key constraints that prevent migration of {@code DomainBase} to SQL databases.
+   *
+   * <p>See {@link InitSqlPipeline} for more information.
+   */
+  static class RemoveDomainBaseForeignKeys extends DoFn<VersionedEntity, VersionedEntity> {
+
+    @ProcessElement
+    public void processElement(
+        @Element VersionedEntity domainBase, OutputReceiver<VersionedEntity> out) {
+      checkArgument(
+          domainBase.getEntity().isPresent(), "Unexpected delete entity %s", domainBase.key());
+      Entity outputEntity =
+          DomainBaseUtil.removeBillingAndPollAndHosts(domainBase.getEntity().get());
+      out.output(
+          VersionedEntity.from(
+              domainBase.commitTimeMills(),
+              EntityTranslator.convertToPb(outputEntity).toByteArray()));
+    }
+  }
 }

core/src/main/java/google/registry/beam/invoicing/BillingEvent.java

@@ -235,6 +235,7 @@ public abstract class BillingEvent implements Serializable {
                 DATE_TIME_FORMATTER.format(eventTime()),
                 registrarId(),
                 billingId(),
+                poNumber(),
                 tld(),
                 action(),
                 domain(),

core/src/main/java/google/registry/beam/spec11/SafeBrowsingTransforms.java

@@ -141,7 +141,7 @@ public class SafeBrowsingTransforms {
     @ProcessElement
     public void processElement(ProcessContext context) {
       Subdomain subdomain = context.element();
-      subdomainBuffer.put(subdomain.fullyQualifiedDomainName(), subdomain);
+      subdomainBuffer.put(subdomain.domainName(), subdomain);
       if (subdomainBuffer.size() >= BATCH_SIZE) {
         ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
         results.forEach(context::output);
@@ -239,7 +239,7 @@ public class SafeBrowsingTransforms {
             String url = match.getJSONObject("threat").getString("url");
             Subdomain subdomain = subdomainBuffer.get(url);
             resultBuilder.add(
-                KV.of(subdomain, ThreatMatch.create(match, subdomain.fullyQualifiedDomainName())));
+                KV.of(subdomain, ThreatMatch.create(match, subdomain.domainName())));
           }
         }
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -77,7 +77,7 @@ public class Spec11Pipeline implements Serializable {
   public static final String REGISTRAR_EMAIL_FIELD = "registrarEmailAddress";
   /** The JSON object field into which we put the registrar's name for Spec11 reports. */
   public static final String REGISTRAR_CLIENT_ID_FIELD = "registrarClientId";
-  /** The JSON object field we put the threat match array for Spec11 reports. */
+  /** The JSON object field into which we put the threat match array for Spec11 reports. */
   public static final String THREAT_MATCHES_FIELD = "threatMatches";
 
   private final String projectId;
@@ -94,8 +94,7 @@ public class Spec11Pipeline implements Serializable {
       @Config("spec11TemplateUrl") String spec11TemplateUrl,
       @Config("reportingBucketUrl") String reportingBucketUrl,
       @LocalCredential GoogleCredentialsBundle googleCredentialsBundle,
-      Retrier retrier
-  ) {
+      Retrier retrier) {
     this.projectId = projectId;
     this.beamStagingUrl = beamStagingUrl;
     this.spec11TemplateUrl = spec11TemplateUrl;
@@ -177,9 +176,11 @@ public class Spec11Pipeline implements Serializable {
       PCollection<Subdomain> domains,
       EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
       ValueProvider<String> dateProvider) {
-    PCollection<KV<Subdomain, ThreatMatch>> subdomains =
+
+    /* Store ThreatMatch objects in JSON. */
+    PCollection<KV<Subdomain, ThreatMatch>> subdomainsJson =
         domains.apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn));
-    subdomains
+    subdomainsJson
         .apply(
             "Map registrar client ID to email/ThreatMatch pair",
             MapElements.into(
@@ -188,7 +189,7 @@ public class Spec11Pipeline implements Serializable {
                 .via(
                     (KV<Subdomain, ThreatMatch> kv) ->
                         KV.of(
-                            kv.getKey().registrarClientId(),
+                            kv.getKey().registrarId(),
                             EmailAndThreatMatch.create(
                                 kv.getKey().registrarEmailAddress(), kv.getValue()))))
         .apply("Group by registrar client ID", GroupByKey.create())

core/src/main/java/google/registry/beam/spec11/Subdomain.java

@@ -36,12 +36,14 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 public abstract class Subdomain implements Serializable {
 
   private static final ImmutableList<String> FIELD_NAMES =
-      ImmutableList.of("fullyQualifiedDomainName", "registrarClientId", "registrarEmailAddress");
+      ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress");
 
   /** Returns the fully qualified domain name. */
-  abstract String fullyQualifiedDomainName();
-  /** Returns the client ID of the associated registrar for this domain. */
-  abstract String registrarClientId();
+  abstract String domainName();
+  /** Returns the domain repo ID (the primary key of the domain table). */
+  abstract String domainRepoId();
+  /** Returns the registrar ID of the associated registrar for this domain. */
+  abstract String registrarId();
   /** Returns the email address of the registrar associated with this domain. */
   abstract String registrarEmailAddress();
 
@@ -56,8 +58,9 @@ public abstract class Subdomain implements Serializable {
     checkFieldsNotNull(FIELD_NAMES, schemaAndRecord);
     GenericRecord record = schemaAndRecord.getRecord();
     return create(
-        extractField(record, "fullyQualifiedDomainName"),
-        extractField(record, "registrarClientId"),
+        extractField(record, "domainName"),
+        extractField(record, "domainRepoId"),
+        extractField(record, "registrarId"),
         extractField(record, "registrarEmailAddress"));
   }
 
@@ -69,9 +72,11 @@ public abstract class Subdomain implements Serializable {
    */
   @VisibleForTesting
   static Subdomain create(
-      String fullyQualifiedDomainName, String registrarClientId, String registrarEmailAddress) {
+      String domainName,
+      String domainRepoId,
+      String registrarId,
+      String registrarEmailAddress) {
     return new AutoValue_Subdomain(
-        fullyQualifiedDomainName, registrarClientId, registrarEmailAddress);
+        domainName, domainRepoId, registrarId, registrarEmailAddress);
   }
 }
-

core/src/main/java/google/registry/beam/spec11/sql/subdomains.sql

@@ -19,11 +19,13 @@
   -- email address.
 
 SELECT
-  domain.fullyQualifiedDomainName AS fullyQualifiedDomainName,
-  registrar.clientId AS registrarClientId,
+  domain.fullyQualifiedDomainName AS domainName,
+  domain.__key__.name AS domainRepoId,
+  registrar.clientId AS registrarId,
   COALESCE(registrar.emailAddress, '') AS registrarEmailAddress
 FROM ( (
     SELECT
+      __key__,
       fullyQualifiedDomainName,
       currentSponsorClientId,
       creationTime

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -367,6 +367,12 @@
     <url-pattern>/_dr/task/linkRdeHosts</url-pattern>
   </servlet-mapping>
 
+  <!-- Action to automatically re-lock a domain after unlocking it -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/relockDomain</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

core/src/main/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml

@@ -7,7 +7,7 @@
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <manual-scaling>
-    <instances>10</instances>
+    <instances>20</instances>
   </manual-scaling>
 
   <system-properties>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -18,7 +18,14 @@
       and streams it to cloud storage. When this job has finished successfully, it'll
       launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
     </description>
-    <schedule>every day 00:07</schedule>
+    <!--
+      This only needs to run once per day, but we launch additional jobs in case the
+      cursor is lagging behind, so it'll catch up to the current date eventually.
+
+      See <a href="../../../production/default/WEB-INF/cron.xml">production config</a> for an
+      explanation of job starting times.
+    -->
+    <schedule>every 12 hours from 00:07 to 12:07</schedule>
     <target>backend</target>
   </cron>
 

core/src/main/java/google/registry/flows/contact/ContactCheckFlow.java

@@ -19,6 +19,7 @@ import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.model.EppResourceUtils.checkResourcesExist;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.flows.EppException;
 import google.registry.flows.ExtensionManager;
@@ -33,8 +34,6 @@ import google.registry.model.eppoutput.CheckData.ContactCheckData;
 import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.util.Clock;
-import java.util.List;
-import java.util.Set;
 import javax.inject.Inject;
 
 /**
@@ -59,9 +58,10 @@ public final class ContactCheckFlow implements Flow {
   public final EppResponse run() throws EppException {
     extensionManager.validate();  // There are no legal extensions for this flow.
     validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
+    ImmutableList<String> targetIds = ((Check) resourceCommand).getTargetIds();
     verifyTargetIdCount(targetIds, maxChecks);
-    Set<String> existingIds = checkResourcesExist(ContactResource.class, targetIds, clock.nowUtc());
+    ImmutableSet<String> existingIds =
+        checkResourcesExist(ContactResource.class, targetIds, clock.nowUtc());
     ImmutableList.Builder<ContactCheck> checks = new ImmutableList.Builder<>();
     for (String id : targetIds) {
       boolean unused = !existingIds.contains(id);

core/src/main/java/google/registry/flows/domain/DomainCheckFlow.java

@@ -15,6 +15,8 @@
 package google.registry.flows.domain;
 
 import static com.google.common.base.Strings.emptyToNull;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.flows.domain.DomainFlowUtils.checkAllowedAccessToTld;
@@ -26,7 +28,6 @@ import static google.registry.flows.domain.DomainFlowUtils.isValidReservedCreate
 import static google.registry.flows.domain.DomainFlowUtils.validateDomainName;
 import static google.registry.flows.domain.DomainFlowUtils.validateDomainNameWithIdnTables;
 import static google.registry.flows.domain.DomainFlowUtils.verifyNotInPredelegation;
-import static google.registry.model.EppResourceUtils.checkResourcesExist;
 import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
 import static google.registry.model.registry.label.ReservationType.getTypeOfHighestSeverity;
 
@@ -48,11 +49,14 @@ import google.registry.flows.custom.DomainCheckFlowCustomLogic.BeforeResponsePar
 import google.registry.flows.custom.DomainCheckFlowCustomLogic.BeforeResponseReturnData;
 import google.registry.flows.domain.token.AllocationTokenDomainCheckResults;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils;
+import google.registry.model.EppResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainCommand.Check;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
 import google.registry.model.domain.fee.FeeCheckCommandExtensionItem;
 import google.registry.model.domain.fee.FeeCheckResponseExtensionItem;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
+import google.registry.model.domain.fee06.FeeCheckCommandExtensionV06;
 import google.registry.model.domain.launch.LaunchCheckExtension;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationTokenExtension;
@@ -62,14 +66,14 @@ import google.registry.model.eppoutput.CheckData.DomainCheck;
 import google.registry.model.eppoutput.CheckData.DomainCheckData;
 import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.eppoutput.EppResponse.ResponseExtension;
+import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.model.registry.label.ReservationType;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
+import google.registry.persistence.VKey;
 import google.registry.util.Clock;
 import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import javax.inject.Inject;
@@ -110,14 +114,20 @@ public final class DomainCheckFlow implements Flow {
   @Inject ExtensionManager extensionManager;
   @Inject EppInput eppInput;
   @Inject @ClientId String clientId;
-  @Inject @Config("maxChecks") int maxChecks;
+
+  @Inject
+  @Config("maxChecks")
+  int maxChecks;
+
   @Inject @Superuser boolean isSuperuser;
   @Inject Clock clock;
   @Inject EppResponse.Builder responseBuilder;
   @Inject AllocationTokenFlowUtils allocationTokenFlowUtils;
   @Inject DomainCheckFlowCustomLogic flowCustomLogic;
   @Inject DomainPricingLogic pricingLogic;
-  @Inject DomainCheckFlow() {}
+
+  @Inject
+  DomainCheckFlow() {}
 
   @Override
   public EppResponse run() throws EppException {
@@ -126,39 +136,41 @@ public final class DomainCheckFlow implements Flow {
     flowCustomLogic.beforeValidation();
     extensionManager.validate();
     validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
+    ImmutableList<String> domainNames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(domainNames, maxChecks);
     DateTime now = clock.nowUtc();
-    ImmutableMap.Builder<String, InternetDomainName> domains = new ImmutableMap.Builder<>();
+    ImmutableMap.Builder<String, InternetDomainName> parsedDomainsBuilder =
+        new ImmutableMap.Builder<>();
     // Only check that the registrar has access to a TLD the first time it is encountered
     Set<String> seenTlds = new HashSet<>();
-    for (String targetId : ImmutableSet.copyOf(targetIds)) {
-      InternetDomainName domainName = validateDomainName(targetId);
-      validateDomainNameWithIdnTables(domainName);
+    for (String domainName : ImmutableSet.copyOf(domainNames)) {
+      InternetDomainName parsedDomain = validateDomainName(domainName);
+      validateDomainNameWithIdnTables(parsedDomain);
       // This validation is moderately expensive, so cache the results.
-      domains.put(targetId, domainName);
-      String tld = domainName.parent().toString();
+      parsedDomainsBuilder.put(domainName, parsedDomain);
+      String tld = parsedDomain.parent().toString();
       boolean tldFirstTimeSeen = seenTlds.add(tld);
       if (tldFirstTimeSeen && !isSuperuser) {
         checkAllowedAccessToTld(clientId, tld);
         verifyNotInPredelegation(Registry.get(tld), now);
       }
     }
-    ImmutableMap<String, InternetDomainName> domainNames = domains.build();
+    ImmutableMap<String, InternetDomainName> parsedDomains = parsedDomainsBuilder.build();
     flowCustomLogic.afterValidation(
         DomainCheckFlowCustomLogic.AfterValidationParameters.newBuilder()
-            .setDomainNames(domainNames)
+            .setDomainNames(parsedDomains)
             // TODO: Use as of date from fee extension v0.12 instead of now, if specified.
             .setAsOfDate(now)
             .build());
-    Set<String> existingIds = checkResourcesExist(DomainBase.class, targetIds, now);
+    ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains =
+        ForeignKeyIndex.load(DomainBase.class, domainNames, now);
     Optional<AllocationTokenExtension> allocationTokenExtension =
         eppInput.getSingleExtension(AllocationTokenExtension.class);
     Optional<AllocationTokenDomainCheckResults> tokenDomainCheckResults =
         allocationTokenExtension.map(
             tokenExtension ->
                 allocationTokenFlowUtils.checkDomainsWithToken(
-                    ImmutableList.copyOf(domainNames.values()),
+                    ImmutableList.copyOf(parsedDomains.values()),
                     tokenExtension.getAllocationToken(),
                     clientId,
                     now));
@@ -173,18 +185,18 @@ public final class DomainCheckFlow implements Flow {
             .orElse(ImmutableMap.of());
     Optional<AllocationToken> allocationToken =
         tokenDomainCheckResults.flatMap(AllocationTokenDomainCheckResults::token);
-    for (String targetId : targetIds) {
+    for (String domainName : domainNames) {
       Optional<String> message =
           getMessageForCheck(
-              domainNames.get(targetId),
-              existingIds,
+              parsedDomains.get(domainName),
+              existingDomains,
               domainCheckResults,
               tldStates,
               allocationToken);
       boolean isAvailable = !message.isPresent();
-      checksBuilder.add(DomainCheck.create(isAvailable, targetId, message.orElse(null)));
+      checksBuilder.add(DomainCheck.create(isAvailable, domainName, message.orElse(null)));
       if (isAvailable) {
-        availableDomains.add(targetId);
+        availableDomains.add(domainName);
       }
     }
     BeforeResponseReturnData responseData =
@@ -193,7 +205,11 @@ public final class DomainCheckFlow implements Flow {
                 .setDomainChecks(checksBuilder.build())
                 .setResponseExtensions(
                     getResponseExtensions(
-                        domainNames, availableDomains.build(), now, allocationToken))
+                        parsedDomains,
+                        existingDomains,
+                        availableDomains.build(),
+                        now,
+                        allocationToken))
                 .setAsOfDate(now)
                 .build());
     return responseBuilder
@@ -204,11 +220,11 @@ public final class DomainCheckFlow implements Flow {
 
   private Optional<String> getMessageForCheck(
       InternetDomainName domainName,
-      Set<String> existingIds,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains,
       ImmutableMap<InternetDomainName, String> tokenCheckResults,
-      Map<String, TldState> tldStates,
+      ImmutableMap<String, TldState> tldStates,
       Optional<AllocationToken> allocationToken) {
-    if (existingIds.contains(domainName.toString())) {
+    if (existingDomains.containsKey(domainName.toString())) {
       return Optional.of("In use");
     }
     TldState tldState = tldStates.get(domainName.parent().toString());
@@ -228,6 +244,7 @@ public final class DomainCheckFlow implements Flow {
   /** Handle the fee check extension. */
   private ImmutableList<? extends ResponseExtension> getResponseExtensions(
       ImmutableMap<String, InternetDomainName> domainNames,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains,
       ImmutableSet<String> availableDomains,
       DateTime now,
       Optional<AllocationToken> allocationToken)
@@ -240,6 +257,9 @@ public final class DomainCheckFlow implements Flow {
     FeeCheckCommandExtension<?, ?> feeCheck = feeCheckOpt.get();
     ImmutableList.Builder<FeeCheckResponseExtensionItem> responseItems =
         new ImmutableList.Builder<>();
+    ImmutableMap<String, EppResource> domainObjs =
+        loadDomainsForRestoreChecks(feeCheck, domainNames, existingDomains);
+
     for (FeeCheckCommandExtensionItem feeCheckItem : feeCheck.getItems()) {
       for (String domainName : getDomainNamesToCheckForFee(feeCheckItem, domainNames.keySet())) {
         FeeCheckResponseExtensionItem.Builder<?> builder = feeCheckItem.createResponseBuilder();
@@ -247,7 +267,7 @@ public final class DomainCheckFlow implements Flow {
             feeCheckItem,
             builder,
             domainNames.get(domainName),
-            Optional.empty(),
+            Optional.ofNullable((DomainBase) domainObjs.get(domainName)),
             feeCheck.getCurrency(),
             now,
             pricingLogic,
@@ -259,12 +279,57 @@ public final class DomainCheckFlow implements Flow {
     return ImmutableList.of(feeCheck.createResponse(responseItems.build()));
   }
 
+  /**
+   * Loads and returns all existing domains that are having restore fees checked.
+   *
+   * <p>This is necessary so that we can check their expiration dates to determine if a one-year
+   * renewal is part of the cost of a restore.
+   *
+   * <p>This may be resource-intensive for large checks of many restore fees, but those are
+   * comparatively rare, and we are at least using an in-memory cache. Also this will get a lot
+   * nicer in Cloud SQL when we can SELECT just the fields we want rather than having to load the
+   * entire entity.
+   */
+  private ImmutableMap<String, EppResource> loadDomainsForRestoreChecks(
+      FeeCheckCommandExtension<?, ?> feeCheck,
+      ImmutableMap<String, InternetDomainName> domainNames,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains) {
+    ImmutableList<String> restoreCheckDomains;
+    if (feeCheck instanceof FeeCheckCommandExtensionV06) {
+      // The V06 fee extension supports specifying the command fees to check on a per-domain basis.
+      restoreCheckDomains =
+          feeCheck.getItems().stream()
+              .filter(fc -> fc.getCommandName() == CommandName.RESTORE)
+              .map(FeeCheckCommandExtensionItem::getDomainName)
+              .collect(toImmutableList());
+    } else if (feeCheck.getItems().stream()
+        .anyMatch(fc -> fc.getCommandName() == CommandName.RESTORE)) {
+      // The more recent fee extension versions support specifying the command fees to check only on
+      // the overall domain check, not per-domain.
+      restoreCheckDomains = ImmutableList.copyOf(domainNames.keySet());
+    } else {
+      // Fall-through case for more recent fee extension versions when the restore fee isn't being
+      // checked.
+      restoreCheckDomains = ImmutableList.of();
+    }
+
+    // Filter down to just domains we know exist and then use the EppResource cache to load them.
+    ImmutableMap<String, VKey<DomainBase>> existingDomainsToLoad =
+        restoreCheckDomains.stream()
+            .filter(existingDomains::containsKey)
+            .collect(toImmutableMap(d -> d, d -> existingDomains.get(d).getResourceKey()));
+    ImmutableMap<VKey<? extends EppResource>, EppResource> loadedDomains =
+        EppResource.loadCached(ImmutableList.copyOf(existingDomainsToLoad.values()));
+    return ImmutableMap.copyOf(
+        Maps.transformEntries(existingDomainsToLoad, (k, v) -> loadedDomains.get(v)));
+  }
+
   /**
    * Return the domains to be checked for a particular fee check item. Some versions of the fee
    * extension specify the domain name in the extension item, while others use the list of domain
    * names from the regular check domain availability list.
    */
-  private Set<String> getDomainNamesToCheckForFee(
+  private ImmutableSet<String> getDomainNamesToCheckForFee(
       FeeCheckCommandExtensionItem feeCheckItem, ImmutableSet<String> availabilityCheckDomains)
       throws OnlyCheckedNamesCanBeFeeCheckedException {
     if (feeCheckItem.isDomainNameSupported()) {

core/src/main/java/google/registry/flows/domain/DomainClaimsCheckFlow.java

@@ -48,7 +48,6 @@ import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.model.tmch.ClaimsListShard;
 import google.registry.util.Clock;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import javax.inject.Inject;
@@ -87,14 +86,14 @@ public final class DomainClaimsCheckFlow implements Flow {
     if (eppInput.getSingleExtension(AllocationTokenExtension.class).isPresent()) {
       throw new DomainClaimsCheckNotAllowedWithAllocationTokens();
     }
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
+    ImmutableList<String> domainNames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(domainNames, maxChecks);
     Set<String> seenTlds = new HashSet<>();
     ImmutableList.Builder<LaunchCheck> launchChecksBuilder = new ImmutableList.Builder<>();
-    for (String targetId : ImmutableSet.copyOf(targetIds)) {
-      InternetDomainName domainName = validateDomainName(targetId);
-      validateDomainNameWithIdnTables(domainName);
-      String tld = domainName.parent().toString();
+    for (String domainName : ImmutableSet.copyOf(domainNames)) {
+      InternetDomainName parsedDomain = validateDomainName(domainName);
+      validateDomainNameWithIdnTables(parsedDomain);
+      String tld = parsedDomain.parent().toString();
       // Only validate access to a TLD the first time it is encountered.
       if (seenTlds.add(tld)) {
         if (!isSuperuser) {
@@ -105,10 +104,10 @@ public final class DomainClaimsCheckFlow implements Flow {
           verifyClaimsPeriodNotEnded(registry, now);
         }
       }
-      Optional<String> claimKey = ClaimsListShard.get().getClaimKey(domainName.parts().get(0));
+      Optional<String> claimKey = ClaimsListShard.get().getClaimKey(parsedDomain.parts().get(0));
       launchChecksBuilder.add(
           LaunchCheck.create(
-              LaunchCheckName.create(claimKey.isPresent(), targetId), claimKey.orElse(null)));
+              LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null)));
     }
     return responseBuilder
         .setOnlyExtension(LaunchCheckResponseExtension.create(CLAIMS, launchChecksBuilder.build()))

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -17,6 +17,7 @@ package google.registry.flows.domain;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Predicates.equalTo;
+import static com.google.common.base.Strings.emptyToNull;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
@@ -32,7 +33,9 @@ import static google.registry.model.registry.Registry.TldState.GENERAL_AVAILABIL
 import static google.registry.model.registry.Registry.TldState.PREDELEGATION;
 import static google.registry.model.registry.Registry.TldState.QUIET_PERIOD;
 import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
+import static google.registry.model.registry.label.ReservationType.ALLOWED_IN_SUNRISE;
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
+import static google.registry.model.registry.label.ReservationType.NAME_COLLISION;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_ANCHOR_TENANT;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_SPECIFIC_USE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -87,6 +90,7 @@ import google.registry.model.domain.DomainCommand.InvalidReferencesException;
 import google.registry.model.domain.DomainCommand.Update;
 import google.registry.model.domain.ForeignKeyedDesignatedContact;
 import google.registry.model.domain.Period;
+import google.registry.model.domain.fee.BaseFee;
 import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Credit;
 import google.registry.model.domain.fee.Fee;
@@ -152,9 +156,7 @@ public class DomainFlowUtils {
 
   /** Reservation types that are only allowed in sunrise by policy. */
   public static final ImmutableSet<ReservationType> TYPES_ALLOWED_FOR_CREATE_ONLY_IN_SUNRISE =
-      Sets.immutableEnumSet(
-          ReservationType.ALLOWED_IN_SUNRISE,
-          ReservationType.NAME_COLLISION);
+      Sets.immutableEnumSet(ALLOWED_IN_SUNRISE, NAME_COLLISION);
 
   /** Warning message for allocation of collision domains in sunrise. */
   public static final String COLLISION_MESSAGE =
@@ -583,15 +585,15 @@ public class DomainFlowUtils {
     builder
         .setCommand(feeRequest.getCommandName(), feeRequest.getPhase(), feeRequest.getSubphase())
         .setCurrencyIfSupported(registry.getCurrency())
-        .setPeriod(feeRequest.getPeriod())
-        .setClass(pricingLogic.getFeeClass(domainNameString, now).orElse(null));
+        .setPeriod(feeRequest.getPeriod());
 
+    String feeClass = null;
     ImmutableList<Fee> fees = ImmutableList.of();
     switch (feeRequest.getCommandName()) {
       case CREATE:
         // Don't return a create price for reserved names.
         if (isReserved(domainName, isSunrise) && !isAvailable) {
-          builder.setClass("reserved"); // Override whatever class we've set above.
+          feeClass = "reserved";
           builder.setAvailIfSupported(false);
           builder.setReasonIfSupported("reserved");
         } else {
@@ -616,15 +618,11 @@ public class DomainFlowUtils {
           throw new RestoresAreAlwaysForOneYearException();
         }
         builder.setAvailIfSupported(true);
-        // The domain object is present only on domain info commands, not on domain check commands,
-        // because check commands can query up to 50 domains and it isn't performant to load them
-        // all. So, only on info commands can we actually determine if we should include the renewal
-        // fee because the domain needs to have been loaded in order to know its expiration time. We
-        // default to including the renewal fee on domain checks because typically most domains are
-        // deleted during the autorenew grace period and thus if restored will require a renewal,
-        // but this is just a best guess.
+        // Domains that never existed, or that used to exist but have completed the entire deletion
+        // process, don't count as expired for the purposes of requiring an added year of renewal on
+        // restore because they can't be restored in the first place.
         boolean isExpired =
-            !domain.isPresent() || domain.get().getRegistrationExpirationTime().isBefore(now);
+            domain.isPresent() && domain.get().getRegistrationExpirationTime().isBefore(now);
         fees = pricingLogic.getRestorePrice(registry, domainNameString, now, isExpired).getFees();
         break;
       case TRANSFER:
@@ -642,6 +640,23 @@ public class DomainFlowUtils {
         throw new UnknownFeeCommandException(feeRequest.getUnparsedCommandName());
     }
 
+    if (feeClass == null) {
+      // Calculate and set the correct fee class based on whether the name is a collision name or we
+      // are returning any premium fees, but only if the fee class isn't already set (i.e. because
+      // the domain is reserved, which overrides any other classes).
+      boolean isNameCollisionInSunrise =
+          registry.getTldState(now).equals(START_DATE_SUNRISE)
+              && getReservationTypes(domainName).contains(NAME_COLLISION);
+      boolean isPremium = fees.stream().anyMatch(BaseFee::isPremium);
+      feeClass =
+          emptyToNull(
+              Joiner.on('-')
+                  .skipNulls()
+                  .join(
+                      isPremium ? "premium" : null, isNameCollisionInSunrise ? "collision" : null));
+    }
+    builder.setClass(feeClass);
+
     // Set the fees, and based on the validDateRange of the fees, set the notAfterDate.
     if (!fees.isEmpty()) {
       builder.setFees(fees);

core/src/main/java/google/registry/flows/domain/DomainPricingLogic.java

@@ -15,7 +15,6 @@
 package google.registry.flows.domain;
 
 import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
-import static google.registry.pricing.PricingEngineProxy.getDomainFeeClass;
 import static google.registry.pricing.PricingEngineProxy.getPricesForDomainName;
 
 import com.google.common.net.InternetDomainName;
@@ -186,11 +185,6 @@ public final class DomainPricingLogic {
             .build());
   }
 
-  /** Returns the fee class for a given domain and date. */
-  public Optional<String> getFeeClass(String domainName, DateTime dateTime) {
-    return getDomainFeeClass(domainName, dateTime);
-  }
-
   /** Returns the domain create cost with allocation-token-related discounts applied. */
   private Money getDomainCreateCostWithDiscount(
       DomainPrices domainPrices, int years, Optional<AllocationToken> allocationToken)

core/src/main/java/google/registry/flows/domain/token/AllocationTokenCustomLogic.java

@@ -16,12 +16,12 @@ package google.registry.flows.domain.token;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import com.google.common.net.InternetDomainName;
 import google.registry.flows.EppException;
 import google.registry.model.domain.DomainCommand;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.registry.Registry;
-import java.util.function.Function;
 import org.joda.time.DateTime;
 
 /**
@@ -50,7 +50,6 @@ public class AllocationTokenCustomLogic {
       String clientId,
       DateTime now) {
     // Do nothing.
-    return domainNames.stream()
-        .collect(ImmutableMap.toImmutableMap(Function.identity(), ignored -> ""));
+    return Maps.toMap(domainNames, k -> "");
   }
 }

core/src/main/java/google/registry/flows/host/HostCheckFlow.java

@@ -19,6 +19,7 @@ import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.model.EppResourceUtils.checkResourcesExist;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.flows.EppException;
 import google.registry.flows.ExtensionManager;
@@ -33,8 +34,6 @@ import google.registry.model.host.HostCommand.Check;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.util.Clock;
-import java.util.List;
-import java.util.Set;
 import javax.inject.Inject;
 
 /**
@@ -59,13 +58,14 @@ public final class HostCheckFlow implements Flow {
   public final EppResponse run() throws EppException {
     extensionManager.validate();  // There are no legal extensions for this flow.
     validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
-    Set<String> existingIds = checkResourcesExist(HostResource.class, targetIds, clock.nowUtc());
+    ImmutableList<String> hostnames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(hostnames, maxChecks);
+    ImmutableSet<String> existingIds =
+        checkResourcesExist(HostResource.class, hostnames, clock.nowUtc());
     ImmutableList.Builder<HostCheck> checks = new ImmutableList.Builder<>();
-    for (String id : targetIds) {
-      boolean unused = !existingIds.contains(id);
-      checks.add(HostCheck.create(unused, id, unused ? null : "In use"));
+    for (String hostname : hostnames) {
+      boolean unused = !existingIds.contains(hostname);
+      checks.add(HostCheck.create(unused, hostname, unused ? null : "In use"));
     }
     return responseBuilder.setResData(HostCheckData.create(checks.build())).build();
   }

core/src/main/java/google/registry/flows/poll/PollAckFlow.java

@@ -14,8 +14,8 @@
 
 package google.registry.flows.poll;
 
-import static com.google.common.base.Preconditions.checkState;
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
+import static google.registry.flows.poll.PollFlowUtils.ackPollMessage;
 import static google.registry.flows.poll.PollFlowUtils.getPollMessagesQuery;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_NO_MESSAGES;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -100,27 +100,8 @@ public class PollAckFlow implements TransactionalFlow {
     // This keeps track of whether we should include the current acked message in the updated
     // message count that's returned to the user. The only case where we do so is if an autorenew
     // poll message is acked, but its next event is already ready to be delivered.
-    boolean includeAckedMessageInCount = false;
-    if (pollMessage instanceof PollMessage.OneTime) {
-      // One-time poll messages are deleted once acked.
-      ofy().delete().entity(pollMessage);
-    } else {
-      checkState(pollMessage instanceof PollMessage.Autorenew, "Unknown poll message type");
-      PollMessage.Autorenew autorenewPollMessage = (PollMessage.Autorenew) pollMessage;
+    boolean includeAckedMessageInCount = ackPollMessage(pollMessage);
 
-      // Move the eventTime of this autorenew poll message forward by a year.
-      DateTime nextEventTime = autorenewPollMessage.getEventTime().plusYears(1);
-
-      // If the next event falls within the bounds of the end time, then just update the eventTime
-      // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
-      // autorenew poll message has no more events to deliver and should be deleted.
-      if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
-        ofy().save().entity(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
-        includeAckedMessageInCount = isBeforeOrAt(nextEventTime, now);
-      } else {
-        ofy().delete().entity(autorenewPollMessage);
-      }
-    }
     // We need to return the new queue length. If this was the last message in the queue being
     // acked, then we return a special status code indicating that. Note that the query will
     // include the message being acked.

core/src/main/java/google/registry/flows/poll/PollFlowUtils.java

@@ -14,7 +14,10 @@
 
 package google.registry.flows.poll;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.poll.PollMessage;
@@ -33,4 +36,42 @@ public final class PollFlowUtils {
         .filter("eventTime <=", now.toDate())
         .order("eventTime");
   }
+
+  /**
+   * Acknowledges the given {@link PollMessage} and returns whether we should include the current
+   * acked message in the updated message count that's returned to the user.
+   *
+   * <p>The only case where we do so is if an autorenew poll message is acked, but its next event is
+   * already ready to be delivered.
+   */
+  public static boolean ackPollMessage(PollMessage pollMessage) {
+    checkArgument(
+        isBeforeOrAt(pollMessage.getEventTime(), tm().getTransactionTime()),
+        "Cannot ACK poll message with ID %s because its event time is in the future: %s",
+        pollMessage.getId(),
+        pollMessage.getEventTime());
+    boolean includeAckedMessageInCount = false;
+    if (pollMessage instanceof PollMessage.OneTime) {
+      // One-time poll messages are deleted once acked.
+      tm().delete(pollMessage.createVKey());
+    } else if (pollMessage instanceof PollMessage.Autorenew) {
+      PollMessage.Autorenew autorenewPollMessage = (PollMessage.Autorenew) pollMessage;
+
+      // Move the eventTime of this autorenew poll message forward by a year.
+      DateTime nextEventTime = autorenewPollMessage.getEventTime().plusYears(1);
+
+      // If the next event falls within the bounds of the end time, then just update the eventTime
+      // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
+      // autorenew poll message has no more events to deliver and should be deleted.
+      if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
+        tm().saveNewOrUpdate(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
+        includeAckedMessageInCount = isBeforeOrAt(nextEventTime, tm().getTransactionTime());
+      } else {
+        tm().delete(autorenewPollMessage.createVKey());
+      }
+    } else {
+      throw new IllegalArgumentException("Unknown poll message type: " + pollMessage.getClass());
+    }
+    return includeAckedMessageInCount;
+  }
 }

core/src/main/java/google/registry/keyring/kms/KmsConnectionImpl.java

@@ -25,11 +25,9 @@ import com.google.api.services.cloudkms.v1.model.DecryptRequest;
 import com.google.api.services.cloudkms.v1.model.EncryptRequest;
 import com.google.api.services.cloudkms.v1.model.KeyRing;
 import com.google.api.services.cloudkms.v1.model.UpdateCryptoKeyPrimaryVersionRequest;
-import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyringException;
 import google.registry.util.Retrier;
 import java.io.IOException;
-import javax.inject.Inject;
 
 /** The {@link KmsConnection} which talks to Cloud KMS. */
 class KmsConnectionImpl implements KmsConnection {
@@ -44,12 +42,7 @@ class KmsConnectionImpl implements KmsConnection {
   private final String projectId;
   private final Retrier retrier;
 
-  @Inject
-  KmsConnectionImpl(
-      @Config("cloudKmsProjectId") String projectId,
-      @Config("cloudKmsKeyRing") String kmsKeyRingName,
-      Retrier retrier,
-      CloudKMS kms) {
+  KmsConnectionImpl(String projectId, String kmsKeyRingName, Retrier retrier, CloudKMS kms) {
     this.projectId = projectId;
     this.kmsKeyRingName = kmsKeyRingName;
     this.retrier = retrier;

core/src/main/java/google/registry/keyring/kms/KmsKeyring.java

@@ -21,6 +21,7 @@ import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 
 import com.googlecode.objectify.Key;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeySerializer;
 import google.registry.keyring.api.Keyring;
 import google.registry.keyring.api.KeyringException;
@@ -86,7 +87,7 @@ public class KmsKeyring implements Keyring {
   private final KmsConnection kmsConnection;
 
   @Inject
-  KmsKeyring(KmsConnection kmsConnection) {
+  KmsKeyring(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
     this.kmsConnection = kmsConnection;
   }
 

core/src/main/java/google/registry/keyring/kms/KmsModule.java

@@ -24,6 +24,7 @@ import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.Keyring;
 import google.registry.util.GoogleCredentialsBundle;
+import google.registry.util.Retrier;
 
 /** Dagger module for Cloud KMS. */
 @Module
@@ -32,9 +33,22 @@ public abstract class KmsModule {
   public static final String NAME = "KMS";
 
   @Provides
+  @Config("defaultKms")
   static CloudKMS provideKms(
       @DefaultCredential GoogleCredentialsBundle credentialsBundle,
       @Config("cloudKmsProjectId") String projectId) {
+    return createKms(credentialsBundle, projectId);
+  }
+
+  @Provides
+  @Config("beamKms")
+  static CloudKMS provideBeamKms(
+      @DefaultCredential GoogleCredentialsBundle credentialsBundle,
+      @Config("beamCloudKmsProjectId") String projectId) {
+    return createKms(credentialsBundle, projectId);
+  }
+
+  private static CloudKMS createKms(GoogleCredentialsBundle credentialsBundle, String projectId) {
     return new CloudKMS.Builder(
             credentialsBundle.getHttpTransport(),
             credentialsBundle.getJsonFactory(),
@@ -43,11 +57,28 @@ public abstract class KmsModule {
         .build();
   }
 
+  @Provides
+  @Config("defaultKmsConnection")
+  static KmsConnection provideKmsConnection(
+      @Config("cloudKmsProjectId") String projectId,
+      @Config("cloudKmsKeyRing") String keyringName,
+      Retrier retrier,
+      @Config("defaultKms") CloudKMS defaultKms) {
+    return new KmsConnectionImpl(projectId, keyringName, retrier, defaultKms);
+  }
+
+  @Provides
+  @Config("beamKmsConnection")
+  static KmsConnection provideBeamKmsConnection(
+      @Config("beamCloudKmsProjectId") String projectId,
+      @Config("beamCloudKmsKeyRing") String keyringName,
+      Retrier retrier,
+      @Config("beamKms") CloudKMS defaultKms) {
+    return new KmsConnectionImpl(projectId, keyringName, retrier, defaultKms);
+  }
+
   @Binds
   @IntoMap
   @StringKey(NAME)
   abstract Keyring provideKeyring(KmsKeyring keyring);
-
-  @Binds
-  abstract KmsConnection provideKmsConnection(KmsConnectionImpl kmsConnectionImpl);
 }

core/src/main/java/google/registry/keyring/kms/KmsUpdater.java

@@ -39,6 +39,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableMap;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeySerializer;
 import google.registry.keyring.kms.KmsKeyring.PrivateKeyLabel;
 import google.registry.keyring.kms.KmsKeyring.PublicKeyLabel;
@@ -64,7 +65,7 @@ public final class KmsUpdater {
   private final HashMap<String, byte[]> secretValues;
 
   @Inject
-  public KmsUpdater(KmsConnection kmsConnection) {
+  public KmsUpdater(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
     this.kmsConnection = kmsConnection;
 
     // Use LinkedHashMap to preserve insertion order on update() to simplify testing and debugging

core/src/main/java/google/registry/model/BackupGroupRoot.java

@@ -14,16 +14,21 @@
 
 package google.registry.model;
 
+import com.google.common.annotations.VisibleForTesting;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlTransient;
 
 /**
  * Base class for entities that are the root of a Registry 2.0 entity group that gets enrolled in
  * commit logs for backup purposes.
  *
- * <p>The commit log system needs to preserve the ordering of closely timed mutations to entities
- * in a single entity group. We require an {@link UpdateAutoTimestamp} field on the root of a group
- * so that we can enforce strictly increasing timestamps.
+ * <p>The commit log system needs to preserve the ordering of closely timed mutations to entities in
+ * a single entity group. We require an {@link UpdateAutoTimestamp} field on the root of a group so
+ * that we can enforce strictly increasing timestamps.
  */
+@MappedSuperclass
 public abstract class BackupGroupRoot extends ImmutableObject {
   /**
    * An automatically managed timestamp of when this object was last written to Datastore.
@@ -32,10 +37,14 @@ public abstract class BackupGroupRoot extends ImmutableObject {
    * that this is updated on every save, rather than only in response to an {@code <update>} command
    */
   @XmlTransient
+  // Prevents subclasses from unexpectedly accessing as property (e.g., HostResource), which would
+  // require an unnecessary non-private setter method.
+  @Access(AccessType.FIELD)
+  @VisibleForTesting
   UpdateAutoTimestamp updateTimestamp = UpdateAutoTimestamp.create(null);
 
   /** Get the {@link UpdateAutoTimestamp} for this entity. */
-  public final UpdateAutoTimestamp getUpdateAutoTimestamp() {
+  public UpdateAutoTimestamp getUpdateTimestamp() {
     return updateTimestamp;
   }
 }

core/src/main/java/google/registry/model/EppResource.java

@@ -16,7 +16,6 @@ package google.registry.model;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
@@ -47,7 +46,6 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
-import java.util.stream.StreamSupport;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
@@ -357,16 +355,18 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
         @Override
         public Map<VKey<? extends EppResource>, EppResource> loadAll(
             Iterable<? extends VKey<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> loadAsMap(keys));
+          return tm().doTransactionless(() -> tm().load(keys));
         }
       };
 
   /**
    * A limited size, limited time cache for EPP resource entities.
    *
-   * <p>This is only used to cache contacts and hosts for the purposes of checking whether they are
-   * deleted or in pending delete during a few domain flows. Any operations on contacts and hosts
-   * directly should of course never use the cache.
+   * <p>This is used to cache contacts and hosts for the purposes of checking whether they are
+   * deleted or in pending delete during a few domain flows, and also to cache domains for the
+   * purpose of determining restore fees in domain checks. Any mutating operations directly on EPP
+   * resources should of course never use the cache as they always need perfectly up-to-date
+   * information.
    */
   @NonFinalForTesting
   private static LoadingCache<VKey<? extends EppResource>, EppResource> cacheEppResources =
@@ -395,7 +395,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   public static ImmutableMap<VKey<? extends EppResource>, EppResource> loadCached(
       Iterable<VKey<? extends EppResource>> keys) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return loadAsMap(keys);
+      return tm().load(keys);
     }
     try {
       return cacheEppResources.getAll(keys);
@@ -423,17 +423,4 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
       throw new RuntimeException("Error loading cached EppResources", e.getCause());
     }
   }
-
-  private static ImmutableMap<VKey<? extends EppResource>, EppResource> loadAsMap(
-      Iterable<? extends VKey<? extends EppResource>> keys) {
-    return StreamSupport.stream(keys.spliterator(), false)
-        // It's possible for us to receive the same key more than once which causes
-        // the immutable map build to break with a duplicate key, so we have to ensure key
-        // uniqueness.
-        .distinct()
-        // We have to use "key -> key" here instead of the identity() function, because
-        // the latter breaks the fairly complicated generic type checking required by the
-        // caching interface.
-        .collect(toImmutableMap(key -> key, key -> tm().load(key)));
-  }
 }

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -23,6 +23,7 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.Result;
@@ -45,7 +46,6 @@ import google.registry.model.transfer.TransferStatus;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
 import javax.annotation.Nullable;
 import org.joda.time.DateTime;
@@ -155,7 +155,7 @@ public final class EppResourceUtils {
     // time for writes.
     return Optional.of(
         cloneProjectedAtTime(
-            resource, latestOf(now, resource.getUpdateAutoTimestamp().getTimestamp())));
+            resource, latestOf(now, resource.getUpdateTimestamp().getTimestamp())));
   }
 
   /**
@@ -169,7 +169,7 @@ public final class EppResourceUtils {
    * @param uniqueIds a list of ids to match
    * @param now the logical time of the check
    */
-  public static <T extends EppResource> Set<String> checkResourcesExist(
+  public static <T extends EppResource> ImmutableSet<String> checkResourcesExist(
       Class<T> clazz, List<String> uniqueIds, final DateTime now) {
     return ForeignKeyIndex.load(clazz, uniqueIds, now).keySet();
   }
@@ -298,7 +298,7 @@ public final class EppResourceUtils {
     // and returns it projected forward to exactly the desired timestamp, or null if the resource is
     // deleted at that timestamp.
     final Result<T> loadResult =
-        isAtOrAfter(timestamp, resource.getUpdateAutoTimestamp().getTimestamp())
+        isAtOrAfter(timestamp, resource.getUpdateTimestamp().getTimestamp())
             ? new ResultNow<>(resource)
             : loadMostRecentRevisionAtTime(resource, timestamp);
     return () -> {

core/src/main/java/google/registry/model/contact/ContactBase.java

@@ -0,0 +1,398 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.contact;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
+
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.IgnoreSave;
+import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.condition.IfNull;
+import google.registry.model.EppResource;
+import google.registry.model.EppResource.ResourceWithTransferData;
+import google.registry.model.transfer.ContactTransferData;
+import google.registry.persistence.VKey;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.Embedded;
+import javax.persistence.MappedSuperclass;
+import javax.xml.bind.annotation.XmlElement;
+import org.joda.time.DateTime;
+
+/**
+ * A persistable contact resource including mutable and non-mutable fields.
+ *
+ * @see <a href="https://tools.ietf.org/html/rfc5733">RFC 5733</a>
+ *     <p>This class deliberately does not include an {@link javax.persistence.Id} so that any
+ *     foreign-keyed fields can refer to the proper parent entity's ID, whether we're storing this
+ *     in the DB itself or as part of another entity
+ */
+@MappedSuperclass
+@Embeddable
+@Access(AccessType.FIELD)
+public class ContactBase extends EppResource implements ResourceWithTransferData {
+
+  /**
+   * Unique identifier for this contact.
+   *
+   * <p>This is only unique in the sense that for any given lifetime specified as the time range
+   * from (creationTime, deletionTime) there can only be one contact in Datastore with this id.
+   * However, there can be many contacts with the same id and non-overlapping lifetimes.
+   */
+  String contactId;
+
+  /**
+   * Localized postal info for the contact. All contained values must be representable in the 7-bit
+   * US-ASCII character set. Personal info; cleared by {@link ContactResource.Builder#wipeOut}.
+   */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_local_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_local_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_local_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_local_country_code"))
+  })
+  PostalInfo localizedPostalInfo;
+
+  /**
+   * Internationalized postal info for the contact. Personal info; cleared by {@link
+   * ContactResource.Builder#wipeOut}.
+   */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_i18n_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_i18n_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_i18n_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_i18n_country_code"))
+  })
+  PostalInfo internationalizedPostalInfo;
+
+  /**
+   * Contact name used for name searches. This is set automatically to be the internationalized
+   * postal name, or if null, the localized postal name, or if that is null as well, null. Personal
+   * info; cleared by {@link ContactResource.Builder#wipeOut}.
+   */
+  @Index String searchName;
+
+  /** Contact’s voice number. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
+  })
+  ContactPhoneNumber voice;
+
+  /** Contact’s fax number. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
+  })
+  ContactPhoneNumber fax;
+
+  /** Contact’s email address. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  String email;
+
+  /** Authorization info (aka transfer secret) of the contact. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
+    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
+  })
+  ContactAuthInfo authInfo;
+
+  /** Data about any pending or past transfers on this contact. */
+  ContactTransferData transferData;
+
+  /**
+   * The time that this resource was last transferred.
+   *
+   * <p>Can be null if the resource has never been transferred.
+   */
+  DateTime lastTransferTime;
+
+  // If any new fields are added which contain personal information, make sure they are cleared by
+  // the wipeOut() function, so that data is not kept around for deleted contacts.
+
+  /** Disclosure policy. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
+    @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
+    @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
+    @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
+    @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
+    @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
+  })
+  Disclose disclose;
+
+  @Override
+  public VKey<? extends ContactBase> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.create(ContactBase.class, getRepoId(), Key.create(this));
+  }
+
+  public String getContactId() {
+    return contactId;
+  }
+
+  public PostalInfo getLocalizedPostalInfo() {
+    return localizedPostalInfo;
+  }
+
+  public PostalInfo getInternationalizedPostalInfo() {
+    return internationalizedPostalInfo;
+  }
+
+  public String getSearchName() {
+    return searchName;
+  }
+
+  public ContactPhoneNumber getVoiceNumber() {
+    return voice;
+  }
+
+  public ContactPhoneNumber getFaxNumber() {
+    return fax;
+  }
+
+  public String getEmailAddress() {
+    return email;
+  }
+
+  public ContactAuthInfo getAuthInfo() {
+    return authInfo;
+  }
+
+  public Disclose getDisclose() {
+    return disclose;
+  }
+
+  public String getCurrentSponsorClientId() {
+    return getPersistedCurrentSponsorClientId();
+  }
+
+  @Override
+  public ContactTransferData getTransferData() {
+    return Optional.ofNullable(transferData).orElse(ContactTransferData.EMPTY);
+  }
+
+  @Override
+  public DateTime getLastTransferTime() {
+    return lastTransferTime;
+  }
+
+  @Override
+  public String getForeignKey() {
+    return contactId;
+  }
+
+  /**
+   * Postal info for the contact.
+   *
+   * <p>The XML marshalling expects the {@link PostalInfo} objects in a list, but we can't actually
+   * persist them to Datastore that way because Objectify can't handle collections of embedded
+   * objects that themselves contain collections, and there's a list of streets inside. This method
+   * transforms the persisted format to the XML format for marshalling.
+   */
+  @XmlElement(name = "postalInfo")
+  public ImmutableList<PostalInfo> getPostalInfosAsList() {
+    return Stream.of(localizedPostalInfo, internationalizedPostalInfo)
+        .filter(Objects::nonNull)
+        .collect(toImmutableList());
+  }
+
+  @Override
+  public ContactBase cloneProjectedAtTime(DateTime now) {
+    return cloneContactProjectedAtTime(this, now);
+  }
+
+  /**
+   * Clones the contact (or subclass). A separate static method so that we can pass in and return a
+   * T without the compiler complaining.
+   */
+  protected static <T extends ContactBase> T cloneContactProjectedAtTime(T contact, DateTime now) {
+    Builder builder = contact.asBuilder();
+    projectResourceOntoBuilderAtTime(contact, builder, now);
+    return (T) builder.build();
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder<>(clone(this));
+  }
+
+  /** A builder for constructing {@link ContactResource}, since it is immutable. */
+  public static class Builder<T extends ContactBase, B extends Builder<T, B>>
+      extends EppResource.Builder<T, B> implements BuilderWithTransferData<ContactTransferData, B> {
+
+    public Builder() {}
+
+    protected Builder(T instance) {
+      super(instance);
+    }
+
+    public B setContactId(String contactId) {
+      getInstance().contactId = contactId;
+      return thisCastToDerived();
+    }
+
+    public B setLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
+      checkArgument(
+          localizedPostalInfo == null
+              || PostalInfo.Type.LOCALIZED.equals(localizedPostalInfo.getType()));
+      getInstance().localizedPostalInfo = localizedPostalInfo;
+      return thisCastToDerived();
+    }
+
+    public B setInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
+      checkArgument(
+          internationalizedPostalInfo == null
+              || PostalInfo.Type.INTERNATIONALIZED.equals(internationalizedPostalInfo.getType()));
+      getInstance().internationalizedPostalInfo = internationalizedPostalInfo;
+      return thisCastToDerived();
+    }
+
+    public B overlayLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
+      return setLocalizedPostalInfo(
+          getInstance().localizedPostalInfo == null
+              ? localizedPostalInfo
+              : getInstance().localizedPostalInfo.overlay(localizedPostalInfo));
+    }
+
+    public B overlayInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
+      return setInternationalizedPostalInfo(
+          getInstance().internationalizedPostalInfo == null
+              ? internationalizedPostalInfo
+              : getInstance().internationalizedPostalInfo.overlay(internationalizedPostalInfo));
+    }
+
+    public B setVoiceNumber(ContactPhoneNumber voiceNumber) {
+      getInstance().voice = voiceNumber;
+      return thisCastToDerived();
+    }
+
+    public B setFaxNumber(ContactPhoneNumber faxNumber) {
+      getInstance().fax = faxNumber;
+      return thisCastToDerived();
+    }
+
+    public B setEmailAddress(String emailAddress) {
+      getInstance().email = emailAddress;
+      return thisCastToDerived();
+    }
+
+    public B setAuthInfo(ContactAuthInfo authInfo) {
+      getInstance().authInfo = authInfo;
+      return thisCastToDerived();
+    }
+
+    public B setDisclose(Disclose disclose) {
+      getInstance().disclose = disclose;
+      return thisCastToDerived();
+    }
+
+    @Override
+    public B setTransferData(ContactTransferData transferData) {
+      getInstance().transferData = transferData;
+      return thisCastToDerived();
+    }
+
+    @Override
+    public B setLastTransferTime(DateTime lastTransferTime) {
+      getInstance().lastTransferTime = lastTransferTime;
+      return thisCastToDerived();
+    }
+
+    /**
+     * Remove all personally identifying information about a contact.
+     *
+     * <p>This should be used when deleting a contact so that the soft-deleted entity doesn't
+     * contain information that the registrant requested to be deleted.
+     */
+    public B wipeOut() {
+      setEmailAddress(null);
+      setFaxNumber(null);
+      setInternationalizedPostalInfo(null);
+      setLocalizedPostalInfo(null);
+      setVoiceNumber(null);
+      return thisCastToDerived();
+    }
+
+    @Override
+    public T build() {
+      T instance = getInstance();
+      // If TransferData is totally empty, set it to null.
+      if (ContactTransferData.EMPTY.equals(instance.transferData)) {
+        setTransferData(null);
+      }
+      // Set the searchName using the internationalized and localized postal info names.
+      if ((instance.internationalizedPostalInfo != null)
+          && (instance.internationalizedPostalInfo.getName() != null)) {
+        instance.searchName = instance.internationalizedPostalInfo.getName();
+      } else if ((instance.localizedPostalInfo != null)
+          && (instance.localizedPostalInfo.getName() != null)) {
+        instance.searchName = instance.localizedPostalInfo.getName();
+      } else {
+        instance.searchName = null;
+      }
+      return super.build();
+    }
+  }
+}

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -0,0 +1,88 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.contact;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.EppResource;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+
+/**
+ * A persisted history entry representing an EPP modification to a contact.
+ *
+ * <p>In addition to the general history fields (e.g. action time, registrar ID) we also persist a
+ * copy of the host entity at this point in time. We persist a raw {@link ContactBase} so that the
+ * foreign-keyed fields in that class can refer to this object.
+ */
+@Entity
+@javax.persistence.Table(
+    indexes = {
+      @javax.persistence.Index(columnList = "creationTime"),
+      @javax.persistence.Index(columnList = "historyRegistrarId"),
+      @javax.persistence.Index(columnList = "historyType"),
+      @javax.persistence.Index(columnList = "historyModificationTime")
+    })
+public class ContactHistory extends HistoryEntry {
+  // Store ContactBase instead of ContactResource so we don't pick up its @Id
+  ContactBase contactBase;
+
+  @Column(nullable = false)
+  VKey<ContactResource> contactRepoId;
+
+  /** The state of the {@link ContactBase} object at this point in time. */
+  public ContactBase getContactBase() {
+    return contactBase;
+  }
+
+  /** The key to the {@link ContactResource} this is based off of. */
+  public VKey<ContactResource> getContactRepoId() {
+    return contactRepoId;
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder(clone(this));
+  }
+
+  public static class Builder extends HistoryEntry.Builder<ContactHistory, ContactHistory.Builder> {
+
+    public Builder() {}
+
+    public Builder(ContactHistory instance) {
+      super(instance);
+    }
+
+    public Builder setContactBase(ContactBase contactBase) {
+      getInstance().contactBase = contactBase;
+      return this;
+    }
+
+    public Builder setContactRepoId(VKey<ContactResource> contactRepoId) {
+      getInstance().contactRepoId = contactRepoId;
+      contactRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
+      return this;
+    }
+
+    // We can remove this once all HistoryEntries are converted to History objects
+    @Override
+    public Builder setParent(Key<? extends EppResource> parent) {
+      super.setParent(parent);
+      getInstance().contactRepoId = VKey.create(ContactResource.class, parent.getName(), parent);
+      return this;
+    }
+  }
+}

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -14,36 +14,16 @@
 
 package google.registry.model.contact;
 
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
-
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
-import com.googlecode.objectify.annotation.IgnoreSave;
-import com.googlecode.objectify.annotation.Index;
-import com.googlecode.objectify.condition.IfNull;
-import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
-import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
-import google.registry.model.contact.PostalInfo.Type;
-import google.registry.model.transfer.ContactTransferData;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.stream.Stream;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
-import javax.persistence.AttributeOverride;
-import javax.persistence.AttributeOverrides;
-import javax.persistence.Column;
-import javax.persistence.Embedded;
-import javax.xml.bind.annotation.XmlElement;
 import org.joda.time.DateTime;
 
 /**
@@ -66,140 +46,12 @@ import org.joda.time.DateTime;
 @ExternalMessagingName("contact")
 @WithStringVKey
 @Access(AccessType.FIELD)
-public class ContactResource extends EppResource
-    implements DatastoreAndSqlEntity, ForeignKeyedEppResource, ResourceWithTransferData {
-
-  /**
-   * Unique identifier for this contact.
-   *
-   * <p>This is only unique in the sense that for any given lifetime specified as the time range
-   * from (creationTime, deletionTime) there can only be one contact in Datastore with this id.
-   * However, there can be many contacts with the same id and non-overlapping lifetimes.
-   */
-  String contactId;
-
-  /**
-   * Localized postal info for the contact. All contained values must be representable in the 7-bit
-   * US-ASCII character set. Personal info; cleared by {@link Builder#wipeOut}.
-   */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
-    @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
-    @AttributeOverride(
-        name = "address.streetLine1",
-        column = @Column(name = "addr_local_street_line1")),
-    @AttributeOverride(
-        name = "address.streetLine2",
-        column = @Column(name = "addr_local_street_line2")),
-    @AttributeOverride(
-        name = "address.streetLine3",
-        column = @Column(name = "addr_local_street_line3")),
-    @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
-    @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
-    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
-    @AttributeOverride(
-        name = "address.countryCode",
-        column = @Column(name = "addr_local_country_code"))
-  })
-  PostalInfo localizedPostalInfo;
-
-  /**
-   * Internationalized postal info for the contact. Personal info; cleared by {@link
-   * Builder#wipeOut}.
-   */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
-    @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
-    @AttributeOverride(
-        name = "address.streetLine1",
-        column = @Column(name = "addr_i18n_street_line1")),
-    @AttributeOverride(
-        name = "address.streetLine2",
-        column = @Column(name = "addr_i18n_street_line2")),
-    @AttributeOverride(
-        name = "address.streetLine3",
-        column = @Column(name = "addr_i18n_street_line3")),
-    @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
-    @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
-    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
-    @AttributeOverride(
-        name = "address.countryCode",
-        column = @Column(name = "addr_i18n_country_code"))
-  })
-  PostalInfo internationalizedPostalInfo;
-
-  /**
-   * Contact name used for name searches. This is set automatically to be the internationalized
-   * postal name, or if null, the localized postal name, or if that is null as well, null. Personal
-   * info; cleared by {@link Builder#wipeOut}.
-   */
-  @Index
-  String searchName;
-
-  /** Contact’s voice number. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
-    @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
-  })
-  ContactPhoneNumber voice;
-
-  /** Contact’s fax number. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
-    @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
-  })
-  ContactPhoneNumber fax;
-
-  /** Contact’s email address. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  String email;
-
-  /** Authorization info (aka transfer secret) of the contact. */
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
-    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
-  })
-  ContactAuthInfo authInfo;
-
-  /** Data about any pending or past transfers on this contact. */
-  ContactTransferData transferData;
-
-  /**
-   * The time that this resource was last transferred.
-   *
-   * <p>Can be null if the resource has never been transferred.
-   */
-  DateTime lastTransferTime;
-
-  // If any new fields are added which contain personal information, make sure they are cleared by
-  // the wipeOut() function, so that data is not kept around for deleted contacts.
-
-  /** Disclosure policy. */
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
-    @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
-    @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
-    @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
-    @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
-    @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
-  })
-  Disclose disclose;
+public class ContactResource extends ContactBase
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource {
 
   @Override
   public VKey<ContactResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.create(ContactResource.class, getRepoId(), Key.create(this));
   }
 
@@ -210,81 +62,9 @@ public class ContactResource extends EppResource
     return super.getRepoId();
   }
 
-  public String getContactId() {
-    return contactId;
-  }
-
-  public PostalInfo getLocalizedPostalInfo() {
-    return localizedPostalInfo;
-  }
-
-  public PostalInfo getInternationalizedPostalInfo() {
-    return internationalizedPostalInfo;
-  }
-
-  public String getSearchName() {
-    return searchName;
-  }
-
-  public ContactPhoneNumber getVoiceNumber() {
-    return voice;
-  }
-
-  public ContactPhoneNumber getFaxNumber() {
-    return fax;
-  }
-
-  public String getEmailAddress() {
-    return email;
-  }
-
-  public ContactAuthInfo getAuthInfo() {
-    return authInfo;
-  }
-
-  public Disclose getDisclose() {
-    return disclose;
-  }
-
-  public final String getCurrentSponsorClientId() {
-    return getPersistedCurrentSponsorClientId();
-  }
-
-  @Override
-  public ContactTransferData getTransferData() {
-    return Optional.ofNullable(transferData).orElse(ContactTransferData.EMPTY);
-  }
-
-  @Override
-  public DateTime getLastTransferTime() {
-    return lastTransferTime;
-  }
-
-  @Override
-  public String getForeignKey() {
-    return contactId;
-  }
-
-  /**
-   * Postal info for the contact.
-   *
-   * <p>The XML marshalling expects the {@link PostalInfo} objects in a list, but we can't actually
-   * persist them to Datastore that way because Objectify can't handle collections of embedded
-   * objects that themselves contain collections, and there's a list of streets inside. This method
-   * transforms the persisted format to the XML format for marshalling.
-   */
-  @XmlElement(name = "postalInfo")
-  public ImmutableList<PostalInfo> getPostalInfosAsList() {
-    return Stream.of(localizedPostalInfo, internationalizedPostalInfo)
-        .filter(Objects::nonNull)
-        .collect(toImmutableList());
-  }
-
   @Override
   public ContactResource cloneProjectedAtTime(DateTime now) {
-    Builder builder = this.asBuilder();
-    projectResourceOntoBuilderAtTime(this, builder, now);
-    return builder.build();
+    return ContactBase.cloneContactProjectedAtTime(this, now);
   }
 
   @Override
@@ -293,116 +73,12 @@ public class ContactResource extends EppResource
   }
 
   /** A builder for constructing {@link ContactResource}, since it is immutable. */
-  public static class Builder extends EppResource.Builder<ContactResource, Builder>
-      implements BuilderWithTransferData<ContactTransferData, Builder> {
+  public static class Builder extends ContactBase.Builder<ContactResource, Builder> {
 
     public Builder() {}
 
     private Builder(ContactResource instance) {
       super(instance);
     }
-
-    public Builder setContactId(String contactId) {
-      getInstance().contactId = contactId;
-      return this;
-    }
-
-    public Builder setLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
-      checkArgument(localizedPostalInfo == null
-          || Type.LOCALIZED.equals(localizedPostalInfo.getType()));
-      getInstance().localizedPostalInfo = localizedPostalInfo;
-      return this;
-    }
-
-    public Builder setInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
-      checkArgument(internationalizedPostalInfo == null
-          || Type.INTERNATIONALIZED.equals(internationalizedPostalInfo.getType()));
-      getInstance().internationalizedPostalInfo = internationalizedPostalInfo;
-      return this;
-    }
-
-    public Builder overlayLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
-      return setLocalizedPostalInfo(getInstance().localizedPostalInfo == null
-          ? localizedPostalInfo
-          : getInstance().localizedPostalInfo.overlay(localizedPostalInfo));
-    }
-
-    public Builder overlayInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
-      return setInternationalizedPostalInfo(getInstance().internationalizedPostalInfo == null
-          ? internationalizedPostalInfo
-          : getInstance().internationalizedPostalInfo.overlay(internationalizedPostalInfo));
-    }
-
-    public Builder setVoiceNumber(ContactPhoneNumber voiceNumber) {
-      getInstance().voice = voiceNumber;
-      return this;
-    }
-
-    public Builder setFaxNumber(ContactPhoneNumber faxNumber) {
-      getInstance().fax = faxNumber;
-      return this;
-    }
-
-    public Builder setEmailAddress(String emailAddress) {
-      getInstance().email = emailAddress;
-      return this;
-    }
-
-    public Builder setAuthInfo(ContactAuthInfo authInfo) {
-      getInstance().authInfo = authInfo;
-      return this;
-    }
-
-    public Builder setDisclose(Disclose disclose) {
-      getInstance().disclose = disclose;
-      return this;
-    }
-
-    @Override
-    public Builder setTransferData(ContactTransferData transferData) {
-      getInstance().transferData = transferData;
-      return this;
-    }
-
-    @Override
-    public Builder setLastTransferTime(DateTime lastTransferTime) {
-      getInstance().lastTransferTime = lastTransferTime;
-      return thisCastToDerived();
-    }
-
-    /**
-     * Remove all personally identifying information about a contact.
-     *
-     * <p>This should be used when deleting a contact so that the soft-deleted entity doesn't
-     * contain information that the registrant requested to be deleted.
-     */
-    public Builder wipeOut() {
-      setEmailAddress(null);
-      setFaxNumber(null);
-      setInternationalizedPostalInfo(null);
-      setLocalizedPostalInfo(null);
-      setVoiceNumber(null);
-      return this;
-    }
-
-    @Override
-    public ContactResource build() {
-      ContactResource instance = getInstance();
-      // If TransferData is totally empty, set it to null.
-      if (ContactTransferData.EMPTY.equals(instance.transferData)) {
-        setTransferData(null);
-      }
-      // Set the searchName using the internationalized and localized postal info names.
-      if ((instance.internationalizedPostalInfo != null)
-          && (instance.internationalizedPostalInfo.getName() != null)) {
-        instance.searchName = instance.internationalizedPostalInfo.getName();
-      } else if ((instance.localizedPostalInfo != null)
-          && (instance.localizedPostalInfo.getName() != null)) {
-        instance.searchName = instance.localizedPostalInfo.getName();
-      } else {
-        instance.searchName = null;
-      }
-      return super.build();
-    }
   }
 }

core/src/main/java/google/registry/model/domain/DomainCommand.java

@@ -41,7 +41,6 @@ import google.registry.model.eppinput.ResourceCommand.SingleResourceCommand;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.persistence.VKey;
-import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -447,7 +446,8 @@ public class DomainCommand {
   private static <T extends EppResource> ImmutableMap<String, VKey<T>> loadByForeignKeysCached(
       final Set<String> foreignKeys, final Class<T> clazz, final DateTime now)
       throws InvalidReferencesException {
-    Map<String, ForeignKeyIndex<T>> fkis = ForeignKeyIndex.loadCached(clazz, foreignKeys, now);
+    ImmutableMap<String, ForeignKeyIndex<T>> fkis =
+        ForeignKeyIndex.loadCached(clazz, foreignKeys, now);
     if (!fkis.keySet().equals(foreignKeys)) {
       throw new InvalidReferencesException(
           clazz, ImmutableSet.copyOf(difference(foreignKeys, fkis.keySet())));

core/src/main/java/google/registry/model/domain/fee/FeeCheckCommandExtension.java

@@ -15,7 +15,6 @@
 package google.registry.model.domain.fee;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.eppinput.EppInput.CommandExtension;
 import org.joda.money.CurrencyUnit;
 
@@ -40,7 +39,7 @@ public interface FeeCheckCommandExtension<
    */
   CurrencyUnit getCurrency();
 
-  ImmutableSet<C> getItems();
+  ImmutableList<C> getItems();
 
   R createResponse(ImmutableList<? extends FeeCheckResponseExtensionItem> items);
 }

core/src/main/java/google/registry/model/domain/fee06/FeeCheckCommandExtensionV06.java

@@ -17,11 +17,10 @@ package google.registry.model.domain.fee06;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
 import google.registry.model.domain.fee.FeeCheckResponseExtensionItem;
-import java.util.Set;
+import java.util.List;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import org.joda.money.CurrencyUnit;
@@ -34,7 +33,7 @@ public class FeeCheckCommandExtensionV06 extends ImmutableObject
         FeeCheckResponseExtensionV06> {
 
   @XmlElement(name = "domain")
-  Set<FeeCheckCommandExtensionItemV06> items;
+  List<FeeCheckCommandExtensionItemV06> items;
 
   @Override
   public CurrencyUnit getCurrency() {
@@ -42,7 +41,7 @@ public class FeeCheckCommandExtensionV06 extends ImmutableObject
   }
 
   @Override
-  public ImmutableSet<FeeCheckCommandExtensionItemV06> getItems() {
+  public ImmutableList<FeeCheckCommandExtensionItemV06> getItems() {
     return nullToEmptyImmutableCopy(items);
   }
 

core/src/main/java/google/registry/model/domain/fee11/FeeCheckCommandExtensionV11.java

@@ -17,7 +17,6 @@ package google.registry.model.domain.fee11;
 import static com.google.common.base.Preconditions.checkState;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
@@ -69,8 +68,8 @@ public class FeeCheckCommandExtensionV11 extends ImmutableObject
   }
 
   @Override
-  public ImmutableSet<FeeCheckCommandExtensionItemV11> getItems() {
-    return ImmutableSet.of(new FeeCheckCommandExtensionItemV11());
+  public ImmutableList<FeeCheckCommandExtensionItemV11> getItems() {
+    return ImmutableList.of(new FeeCheckCommandExtensionItemV11());
   }
 
   @Override

core/src/main/java/google/registry/model/domain/fee12/FeeCheckCommandExtensionV12.java

@@ -17,11 +17,10 @@ package google.registry.model.domain.fee12;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
 import google.registry.model.domain.fee.FeeCheckResponseExtensionItem;
-import java.util.Set;
+import java.util.List;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlType;
@@ -43,10 +42,10 @@ public class FeeCheckCommandExtensionV12 extends ImmutableObject
   }
 
   @XmlElement(name = "command")
-  Set<FeeCheckCommandExtensionItemV12> items;
+  List<FeeCheckCommandExtensionItemV12> items;
 
   @Override
-  public ImmutableSet<FeeCheckCommandExtensionItemV12> getItems() {
+  public ImmutableList<FeeCheckCommandExtensionItemV12> getItems() {
     return nullToEmptyImmutableCopy(items);
   }
 

core/src/main/java/google/registry/model/domain/rgp/GracePeriodStatus.java

@@ -86,10 +86,7 @@ public enum GracePeriodStatus implements EppEnum {
   /** Provide a quick lookup of GracePeriodStatus from XML name. */
   private static final ImmutableMap<String, GracePeriodStatus> XML_NAME_TO_GRACE_PERIOD_STATUS =
       Stream.of(GracePeriodStatus.values())
-          .collect(
-              toImmutableMap(
-                  (GracePeriodStatus gracePeriodStatus) -> gracePeriodStatus.xmlName,
-                  value -> value));
+          .collect(toImmutableMap(GracePeriodStatus::getXmlName, value -> value));
 
   @XmlAttribute(name = "s")
   private final String xmlName;

core/src/main/java/google/registry/model/host/HostBase.java

@@ -125,7 +125,7 @@ public class HostBase extends EppResource {
   }
 
   @Override
-  public VKey<? extends EppResource> createVKey() {
+  public VKey<? extends HostBase> createVKey() {
     return VKey.create(HostBase.class, getRepoId(), Key.create(this));
   }
 

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -73,7 +73,7 @@ public class HostHistory extends HistoryEntry {
       return this;
     }
 
-    public Builder setHostResourceId(VKey<HostResource> hostRepoId) {
+    public Builder setHostRepoId(VKey<HostResource> hostRepoId) {
       getInstance().hostRepoId = hostRepoId;
       hostRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
       return this;
@@ -83,8 +83,7 @@ public class HostHistory extends HistoryEntry {
     @Override
     public Builder setParent(Key<? extends EppResource> parent) {
       super.setParent(parent);
-      getInstance().hostRepoId =
-          VKey.create(HostResource.class, parent.getName(), (Key<HostResource>) parent);
+      getInstance().hostRepoId = VKey.create(HostResource.class, parent.getName(), parent);
       return this;
     }
   }

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -15,7 +15,6 @@
 package google.registry.model.index;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static com.google.common.collect.Maps.filterValues;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -178,11 +177,11 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * <p>The returned map will omit any keys for which the {@link ForeignKeyIndex} doesn't exist or
    * has been soft deleted.
    */
-  public static <E extends EppResource> Map<String, ForeignKeyIndex<E>> load(
+  public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> load(
       Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
-    return filterValues(
-        ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys),
-        (ForeignKeyIndex<?> fki) -> now.isBefore(fki.deletionTime));
+    return ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys).entrySet().stream()
+        .filter(e -> now.isBefore(e.getValue().deletionTime))
+        .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
   }
 
   static final CacheLoader<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>> CACHE_LOADER =
@@ -249,7 +248,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * <p>Don't use the cached version of this method unless you really need it for performance
    * reasons, and are OK with the trade-offs in loss of transactional consistency.
    */
-  public static <E extends EppResource> Map<String, ForeignKeyIndex<E>> loadCached(
+  public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> loadCached(
       Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
       return tm().doTransactionless(() -> load(clazz, foreignKeys, now));
@@ -262,16 +261,14 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
       // This cast is safe because when we loaded ForeignKeyIndexes above we used type clazz, which
       // is scoped to E.
       @SuppressWarnings("unchecked")
-      Map<String, ForeignKeyIndex<E>> fkisFromCache = cacheForeignKeyIndexes
-          .getAll(fkiKeys)
-          .entrySet()
-          .stream()
-          .filter(entry -> entry.getValue().isPresent())
-          .filter(entry -> now.isBefore(entry.getValue().get().getDeletionTime()))
-          .collect(
-              ImmutableMap.toImmutableMap(
-                  entry -> entry.getKey().getName(),
-                  entry -> (ForeignKeyIndex<E>) entry.getValue().get()));
+      ImmutableMap<String, ForeignKeyIndex<E>> fkisFromCache =
+          cacheForeignKeyIndexes.getAll(fkiKeys).entrySet().stream()
+              .filter(entry -> entry.getValue().isPresent())
+              .filter(entry -> now.isBefore(entry.getValue().get().getDeletionTime()))
+              .collect(
+                  ImmutableMap.toImmutableMap(
+                      entry -> entry.getKey().getName(),
+                      entry -> (ForeignKeyIndex<E>) entry.getValue().get()));
       return fkisFromCache;
     } catch (ExecutionException e) {
       throw new RuntimeException("Error loading cached ForeignKeyIndexes", e.getCause());

core/src/main/java/google/registry/model/ofy/CommitLoggedWork.java

@@ -168,7 +168,7 @@ class CommitLoggedWork<R> implements Runnable {
       DateTime transactionTime, Set<Entry<Key<BackupGroupRoot>, BackupGroupRoot>> bgrEntries) {
     ImmutableMap.Builder<Key<BackupGroupRoot>, DateTime> builder = new ImmutableMap.Builder<>();
     for (Entry<Key<BackupGroupRoot>, BackupGroupRoot> entry : bgrEntries) {
-      DateTime updateTime = entry.getValue().getUpdateAutoTimestamp().getTimestamp();
+      DateTime updateTime = entry.getValue().getUpdateTimestamp().getTimestamp();
       if (!updateTime.isBefore(transactionTime)) {
         builder.put(entry.getKey(), updateTime);
       }

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -15,15 +15,18 @@
 package google.registry.model.ofy;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.base.Functions;
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.TransactionManager;
-import java.util.Iterator;
+import java.util.Map.Entry;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
@@ -156,12 +159,15 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public <T> ImmutableList<T> load(Iterable<VKey<T>> keys) {
-    Iterator<Key<T>> iter =
-        StreamSupport.stream(keys.spliterator(), false).map(VKey::getOfyKey).iterator();
+  public <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys) {
+    // Keep track of the Key -> VKey mapping so we can translate them back.
+    ImmutableMap<Key<T>, VKey<? extends T>> keyMap =
+        StreamSupport.stream(keys.spliterator(), false)
+            .distinct()
+            .collect(toImmutableMap(key -> (Key<T>) key.getOfyKey(), Functions.identity()));
 
-    // The lambda argument to keys() effectively converts Iterator -> Iterable.
-    return ImmutableList.copyOf(getOfy().load().keys(() -> iter).values());
+    return getOfy().load().keys(keyMap.keySet()).entrySet().stream()
+        .collect(ImmutableMap.toImmutableMap(entry -> keyMap.get(entry.getKey()), Entry::getValue));
   }
 
   @Override

core/src/main/java/google/registry/model/pricing/PremiumPricingEngine.java

@@ -14,7 +14,6 @@
 
 package google.registry.model.pricing;
 
-import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
@@ -46,18 +45,12 @@ public interface PremiumPricingEngine {
     // create, renew, restore, and transfer.
     private Money createCost;
     private Money renewCost;
-    private Optional<String> feeClass;
 
-    static DomainPrices create(
-        boolean isPremium,
-        Money createCost,
-        Money renewCost,
-        Optional<String> feeClass) {
+    static DomainPrices create(boolean isPremium, Money createCost, Money renewCost) {
       DomainPrices instance = new DomainPrices();
       instance.isPremium = isPremium;
       instance.createCost = createCost;
       instance.renewCost = renewCost;
-      instance.feeClass = feeClass;
       return instance;
     }
 
@@ -75,10 +68,5 @@ public interface PremiumPricingEngine {
     public Money getRenewCost() {
       return renewCost;
     }
-
-    /** Returns the fee class of the cost (used for the Fee extension). */
-    public Optional<String> getFeeClass() {
-      return feeClass;
-    }
   }
 }

core/src/main/java/google/registry/model/pricing/StaticPremiumListPricingEngine.java

@@ -15,14 +15,9 @@
 package google.registry.model.pricing;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Strings.emptyToNull;
-import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
 import static google.registry.model.registry.label.PremiumListUtils.getPremiumPrice;
-import static google.registry.model.registry.label.ReservationType.NAME_COLLISION;
-import static google.registry.model.registry.label.ReservedList.getReservationTypes;
 import static google.registry.util.DomainNameUtils.getTldFromDomainName;
 
-import com.google.common.base.Joiner;
 import com.google.common.net.InternetDomainName;
 import google.registry.model.registry.Registry;
 import java.util.Optional;
@@ -44,16 +39,9 @@ public final class StaticPremiumListPricingEngine implements PremiumPricingEngin
     String label = InternetDomainName.from(fullyQualifiedDomainName).parts().get(0);
     Registry registry = Registry.get(checkNotNull(tld, "tld"));
     Optional<Money> premiumPrice = getPremiumPrice(label, registry);
-    boolean isNameCollisionInSunrise =
-        registry.getTldState(priceTime).equals(START_DATE_SUNRISE)
-            && getReservationTypes(label, tld).contains(NAME_COLLISION);
-    String feeClass = emptyToNull(Joiner.on('-').skipNulls().join(
-            premiumPrice.isPresent() ? "premium" : null,
-            isNameCollisionInSunrise ? "collision" : null));
     return DomainPrices.create(
         premiumPrice.isPresent(),
         premiumPrice.orElse(registry.getStandardCreateCost()),
-        premiumPrice.orElse(registry.getStandardRenewCost(priceTime)),
-        Optional.ofNullable(feeClass));
+        premiumPrice.orElse(registry.getStandardRenewCost(priceTime)));
   }
 }

core/src/main/java/google/registry/model/rde/RdeRevision.java

@@ -48,6 +48,10 @@ public final class RdeRevision extends ImmutableObject {
    */
   int revision;
 
+  public int getRevision() {
+    return revision;
+  }
+
   /**
    * Returns next revision ID to use when staging a new deposit file for the given triplet.
    *

core/src/main/java/google/registry/model/registry/Registries.java

@@ -19,13 +19,13 @@ import static com.google.common.base.Predicates.equalTo;
 import static com.google.common.base.Predicates.in;
 import static com.google.common.base.Predicates.not;
 import static com.google.common.base.Strings.emptyToNull;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Maps.filterValues;
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.base.Joiner;
@@ -37,7 +37,6 @@ import com.google.common.collect.Streams;
 import com.google.common.net.InternetDomainName;
 import com.googlecode.objectify.Key;
 import google.registry.model.registry.Registry.TldType;
-import java.util.Map;
 import java.util.Optional;
 
 /** Utilities for finding and listing {@link Registry} entities. */
@@ -71,7 +70,7 @@ public final class Registries {
                               .collect(toImmutableSet());
                       return Registry.getAll(tlds).stream()
                           .map(e -> Maps.immutableEntry(e.getTldStr(), e.getTldType()))
-                          .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+                          .collect(entriesToImmutableMap());
                     }));
   }
 

core/src/main/java/google/registry/model/registry/Registry.java

@@ -587,8 +587,9 @@ public class Registry extends ImmutableObject implements Buildable {
     return Fee.create(
         eapFeeSchedule.getValueAtTime(now).getAmount(),
         FeeType.EAP,
-        // An EAP fee counts as premium so the domain's overall Fee doesn't show as standard-priced.
-        true,
+        // An EAP fee does not count as premium -- it's a separate one-time fee, independent of
+        // which the domain is separately considered standard vs premium depending on renewal price.
+        false,
         validPeriod,
         validPeriod.upperEndpoint());
   }

core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java

@@ -15,6 +15,7 @@
 package google.registry.model.registry.label;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
@@ -30,6 +31,7 @@ import com.google.common.collect.Multiset;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
@@ -42,6 +44,11 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -49,25 +56,46 @@ import org.joda.time.DateTime;
  *
  * @param <T> The type of the root value being listed, e.g. {@link ReservationType}.
  * @param <R> The type of domain label entry being listed, e.g. {@link ReservedListEntry} (note,
- *            must subclass {@link DomainLabelEntry}.
+ *     must subclass {@link DomainLabelEntry}.
  */
+@MappedSuperclass
 public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends DomainLabelEntry<T, ?>>
     extends ImmutableObject implements Buildable {
 
+  @Ignore
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  Long revisionId;
+
   @Id
+  @Column(nullable = false)
   String name;
 
-  @Parent
-  Key<EntityGroupRoot> parent = getCrossTldKey();
+  @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
-  DateTime creationTime;
+  @Transient DateTime creationTime;
 
+  // The list in Cloud SQL is immutable, we only have a creation_timestamp field and it should be
+  // set to the timestamp when the list is created. In Datastore, we have two fields and the
+  // lastUpdateTime is set to the current timestamp when creating and updating a list. So, we use
+  // lastUpdateTime as the creation_timestamp column during the dual-write phase for compatibility.
+  @Column(name = "creation_timestamp", nullable = false)
   DateTime lastUpdateTime;
 
+  /** Returns the ID of this revision, or throws if null. */
+  public long getRevisionId() {
+    checkState(
+        revisionId != null,
+        "revisionId is null because this object has not been persisted to the database yet");
+    return revisionId;
+  }
+
+  /** Returns the name of the reserved list. */
   public String getName() {
     return name;
   }
 
+  /** Returns the creation time of this revision of the reserved list. */
   public DateTime getCreationTime() {
     return creationTime;
   }
@@ -183,6 +211,9 @@ public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends Dom
     @Override
     public T build() {
       checkArgument(!isNullOrEmpty(getInstance().name), "List must have a name");
+      // The list is immutable in Cloud SQL, so make sure the revision id is not set when the
+      // builder object is created from a list object
+      getInstance().revisionId = null;
       return super.build();
     }
   }

core/src/main/java/google/registry/model/registry/label/DomainLabelEntry.java

@@ -23,16 +23,20 @@ import com.google.common.net.InternetDomainName;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
+import javax.persistence.Column;
+import javax.persistence.MappedSuperclass;
 
 /**
  * Represents a label entry parsed from a line in a reserved/premium list txt file.
  *
  * @param <T> The type of the value stored for the domain label, e.g. {@link ReservationType}.
  */
+@MappedSuperclass
 public abstract class DomainLabelEntry<T extends Comparable<?>, D extends DomainLabelEntry<?, ?>>
     extends ImmutableObject implements Comparable<D> {
 
   @Id
+  @Column(name = "domain_label", insertable = false, updatable = false)
   String label;
 
   String comment;

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -14,7 +14,9 @@
 
 package google.registry.model.registry.label;
 
+import static com.google.common.base.Charsets.US_ASCII;
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.hash.Funnels.stringFunnel;
 import static com.google.common.hash.Funnels.unencodedCharsFunnel;
 import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
 import static google.registry.config.RegistryConfig.getSingletonCachePersistDuration;
@@ -32,43 +34,82 @@ import com.google.common.cache.CacheLoader;
 import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.hash.BloomFilter;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registry.Registry;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.tld.PremiumListDao;
 import google.registry.util.NonFinalForTesting;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import javax.annotation.Nullable;
+import javax.persistence.CollectionTable;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.MapKeyColumn;
+import javax.persistence.PostLoad;
+import javax.persistence.PrePersist;
+import javax.persistence.Table;
+import javax.persistence.Transient;
+import org.hibernate.LazyInitializationException;
+import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.Duration;
 
-/** A premium list entity, persisted to Datastore, that is used to check domain label prices. */
+/**
+ * A premium list entity that is used to check domain label prices.
+ *
+ * <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
+ * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
+ * succeeds, we will end up with having two exact same premium lists that differ only by revisionId.
+ * This is fine though, because we only use the list with the highest revisionId.
+ */
 @ReportedOn
 @Entity
+@javax.persistence.Entity
+@Table(indexes = {@Index(columnList = "name", name = "premiumlist_name_idx")})
 public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.PremiumListEntry>
-    implements DatastoreEntity {
+    implements DatastoreAndSqlEntity {
 
   /** Stores the revision key for the set of currently used premium list entry entities. */
-  Key<PremiumListRevision> revisionKey;
+  @Transient Key<PremiumListRevision> revisionKey;
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // PremiumList is dual-written
-  }
+  @Ignore
+  @Column(nullable = false)
+  CurrencyUnit currency;
+
+  @Ignore
+  @ElementCollection
+  @CollectionTable(
+      name = "PremiumEntry",
+      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
+  @MapKeyColumn(name = "domainLabel")
+  @Column(name = "price", nullable = false)
+  Map<String, BigDecimal> labelsToPrices;
+
+  @Ignore
+  @Column(nullable = false)
+  BloomFilter<String> bloomFilter;
 
   /** Virtual parent entity for premium list entry entities associated with a single revision. */
   @ReportedOn
@@ -247,6 +288,35 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
     return Optional.ofNullable(loadPremiumList(name));
   }
 
+  /** Returns the {@link CurrencyUnit} used for this list. */
+  public CurrencyUnit getCurrency() {
+    return currency;
+  }
+
+  /**
+   * Returns a {@link Map} of domain labels to prices.
+   *
+   * <p>Note that this is lazily loaded and thus will throw a {@link LazyInitializationException} if
+   * used outside the transaction in which the given entity was loaded. You generally should not be
+   * using this anyway as it's inefficient to load all of the PremiumEntry rows if you don't need
+   * them. To check prices, use {@link PremiumListDao#getPremiumPrice} instead.
+   */
+  @Nullable
+  public ImmutableMap<String, BigDecimal> getLabelsToPrices() {
+    return labelsToPrices == null ? null : ImmutableMap.copyOf(labelsToPrices);
+  }
+
+  /**
+   * Returns a Bloom filter to determine whether a label might be premium, or is definitely not.
+   *
+   * <p>If the domain label might be premium, then the next step is to check for the existence of a
+   * corresponding row in the PremiumListEntry table. Otherwise, we know for sure it's not premium,
+   * and no DB load is required.
+   */
+  public BloomFilter<String> getBloomFilter() {
+    return bloomFilter;
+  }
+
   /**
    * A premium list entry entity, persisted to Datastore. Each instance represents the price of a
    * single label on a given TLD.
@@ -339,9 +409,39 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
       return this;
     }
 
+    public Builder setCurrency(CurrencyUnit currency) {
+      getInstance().currency = currency;
+      return this;
+    }
+
+    public Builder setLabelsToPrices(Map<String, BigDecimal> labelsToPrices) {
+      getInstance().labelsToPrices = ImmutableMap.copyOf(labelsToPrices);
+      return this;
+    }
+
     @Override
     public PremiumList build() {
+      if (getInstance().labelsToPrices != null) {
+        // ASCII is used for the charset because all premium list domain labels are stored
+        // punycoded.
+        getInstance().bloomFilter =
+            BloomFilter.create(stringFunnel(US_ASCII), getInstance().labelsToPrices.size());
+        getInstance()
+            .labelsToPrices
+            .keySet()
+            .forEach(label -> getInstance().bloomFilter.put(label));
+      }
       return super.build();
     }
   }
+
+  @PrePersist
+  void prePersist() {
+    lastUpdateTime = creationTime;
+  }
+
+  @PostLoad
+  void postLoad() {
+    creationTime = lastUpdateTime;
+  }
 }

core/src/main/java/google/registry/model/registry/label/ReservedList.java

@@ -16,11 +16,8 @@ package google.registry.model.registry.label;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
-import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -30,13 +27,8 @@ import com.google.common.base.Splitter;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.MapDifference.ValueDifference;
-import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
@@ -46,45 +38,58 @@ import com.googlecode.objectify.mapper.Mapper;
 import google.registry.model.Buildable;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.label.DomainLabelMetrics.MetricsReservedListMatch;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
-import google.registry.schema.tld.ReservedList.ReservedEntry;
-import google.registry.schema.tld.ReservedListDao;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import javax.annotation.Nullable;
+import javax.persistence.CollectionTable;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.Embeddable;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.MapKeyColumn;
+import javax.persistence.Table;
 import org.joda.time.DateTime;
 
 /**
- * A reserved list entity, persisted to Datastore, that is used to check domain label reservations.
+ * A list of reserved domain labels that are blocked from being registered for various reasons.
+ *
+ * <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
+ * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
+ * succeeds, we will end up with having two exact same reserved lists that differ only by
+ * revisionId. This is fine though, because we only use the list with the highest revisionId.
  */
 @Entity
+@javax.persistence.Entity
+@Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")})
 public final class ReservedList
     extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry>
-    implements DatastoreEntity {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    implements DatastoreAndSqlEntity {
 
   @Mapify(ReservedListEntry.LabelMapper.class)
+  @ElementCollection
+  @CollectionTable(
+      name = "ReservedEntry",
+      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
+  @MapKeyColumn(name = "domain_label")
   Map<String, ReservedListEntry> reservedListMap;
 
+  @Column(nullable = false)
   boolean shouldPublish = true;
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // ReservedList is dual-written
-  }
-
   /**
    * A reserved list entry entity, persisted to Datastore, that represents a single label and its
    * reservation type.
    */
   @Embed
-  public static class ReservedListEntry
-      extends DomainLabelEntry<ReservationType, ReservedListEntry> implements Buildable {
+  @Embeddable
+  public static class ReservedListEntry extends DomainLabelEntry<ReservationType, ReservedListEntry>
+      implements Buildable {
 
+    @Column(nullable = false)
     ReservationType reservationType;
 
     /** Mapper for use with @Mapify */
@@ -151,6 +156,7 @@ public final class ReservedList
     return shouldPublish;
   }
 
+  /** Returns a {@link Map} of domain labels to {@link ReservedListEntry}. */
   public ImmutableMap<String, ReservedListEntry> getReservedListEntries() {
     return ImmutableMap.copyOf(nullToEmpty(reservedListMap));
   }
@@ -240,68 +246,10 @@ public final class ReservedList
               new CacheLoader<String, ReservedList>() {
                 @Override
                 public ReservedList load(String listName) {
-                  ReservedList datastoreList =
-                      ofy()
-                          .load()
-                          .type(ReservedList.class)
-                          .parent(getCrossTldKey())
-                          .id(listName)
-                          .now();
-                  // Also load the list from Cloud SQL, compare the two lists, and log if different.
-                  try {
-                    loadAndCompareCloudSqlList(datastoreList);
-                  } catch (Throwable t) {
-                    logger.atSevere().withCause(t).log("Error comparing reserved lists.");
-                  }
-                  return datastoreList;
+                  return ReservedListDualWriteDao.getLatestRevision(listName).orElse(null);
                 }
               });
 
-  private static final void loadAndCompareCloudSqlList(ReservedList datastoreList) {
-    Optional<google.registry.schema.tld.ReservedList> maybeCloudSqlList =
-        ReservedListDao.getLatestRevision(datastoreList.getName());
-    if (maybeCloudSqlList.isPresent()) {
-      Map<String, ReservedEntry> datastoreLabelsToReservations =
-          datastoreList.reservedListMap.entrySet().parallelStream()
-              .collect(
-                  toImmutableMap(
-                      entry -> entry.getKey(),
-                      entry ->
-                          ReservedEntry.create(
-                              entry.getValue().reservationType, entry.getValue().comment)));
-
-      google.registry.schema.tld.ReservedList cloudSqlList = maybeCloudSqlList.get();
-      MapDifference<String, ReservedEntry> diff =
-          Maps.difference(datastoreLabelsToReservations, cloudSqlList.getLabelsToReservations());
-      if (!diff.areEqual()) {
-        if (diff.entriesDiffering().size() > 10) {
-          logger.atWarning().log(
-              String.format(
-                  "Unequal reserved lists detected, Cloud SQL list with revision"
-                      + " id %d has %d different records than the current"
-                      + " Datastore list.",
-                  cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
-        } else {
-          StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
-          diff.entriesDiffering().entrySet().stream()
-              .forEach(
-                  entry -> {
-                    String label = entry.getKey();
-                    ValueDifference<ReservedEntry> valueDiff = entry.getValue();
-                    diffMessage.append(
-                        String.format(
-                            "Domain label %s has entry %s in Datastore and entry"
-                                + " %s in Cloud SQL.\n",
-                            label, valueDiff.leftValue(), valueDiff.rightValue()));
-                  });
-          logger.atWarning().log(diffMessage.toString());
-        }
-      }
-    } else {
-      logger.atWarning().log("Reserved list in Cloud SQL is empty.");
-    }
-  }
-
   /**
    * Gets the {@link ReservationType} of a label in a single ReservedList, or returns an absent
    * Optional if none exists in the list.

core/src/main/java/google/registry/model/registry/label/ReservedListDualWriteDao.java

@@ -0,0 +1,124 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.registry.label;
+
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.MapDifference.ValueDifference;
+import com.google.common.collect.Maps;
+import com.google.common.flogger.FluentLogger;
+import com.googlecode.objectify.Key;
+import google.registry.model.registry.label.ReservedList.ReservedListEntry;
+import google.registry.persistence.VKey;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A {@link ReservedList} DAO that does dual-write and dual-read against Datastore and Cloud SQL. It
+ * still uses Datastore as the primary storage and suppresses any exception thrown by Cloud SQL.
+ *
+ * <p>TODO(b/160993806): Delete this DAO and switch to use the SQL only DAO after migrating to Cloud
+ * SQL.
+ */
+public class ReservedListDualWriteDao {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private ReservedListDualWriteDao() {}
+
+  /** Persist a new reserved list to Cloud SQL. */
+  public static void save(ReservedList reservedList) {
+    ofyTm().transact(() -> ofyTm().saveNewOrUpdate(reservedList));
+    try {
+      logger.atInfo().log("Saving reserved list %s to Cloud SQL", reservedList.getName());
+      ReservedListSqlDao.save(reservedList);
+      logger.atInfo().log(
+          "Saved reserved list %s with %d entries to Cloud SQL",
+          reservedList.getName(), reservedList.getReservedListEntries().size());
+    } catch (Throwable t) {
+      logger.atSevere().withCause(t).log("Error saving the reserved list to Cloud SQL.");
+    }
+  }
+
+  /**
+   * Returns the most recent revision of the {@link ReservedList} with the specified name, if it
+   * exists.
+   */
+  public static Optional<ReservedList> getLatestRevision(String reservedListName) {
+    Optional<ReservedList> maybeDatastoreList =
+        ofyTm()
+            .maybeLoad(
+                VKey.createOfy(
+                    ReservedList.class,
+                    Key.create(getCrossTldKey(), ReservedList.class, reservedListName)));
+    try {
+      // Also load the list from Cloud SQL, compare the two lists, and log if different.
+      maybeDatastoreList.ifPresent(ReservedListDualWriteDao::loadAndCompareCloudSqlList);
+    } catch (Throwable t) {
+      logger.atSevere().withCause(t).log("Error comparing reserved lists.");
+    }
+    return maybeDatastoreList;
+  }
+
+  private static void loadAndCompareCloudSqlList(ReservedList datastoreList) {
+    Optional<ReservedList> maybeCloudSqlList =
+        ReservedListSqlDao.getLatestRevision(datastoreList.getName());
+    if (maybeCloudSqlList.isPresent()) {
+      Map<String, ReservedListEntry> datastoreLabelsToReservations =
+          datastoreList.reservedListMap.entrySet().parallelStream()
+              .collect(
+                  toImmutableMap(
+                      Map.Entry::getKey,
+                      entry ->
+                          ReservedListEntry.create(
+                              entry.getKey(),
+                              entry.getValue().reservationType,
+                              entry.getValue().comment)));
+
+      ReservedList cloudSqlList = maybeCloudSqlList.get();
+      MapDifference<String, ReservedListEntry> diff =
+          Maps.difference(datastoreLabelsToReservations, cloudSqlList.reservedListMap);
+      if (!diff.areEqual()) {
+        if (diff.entriesDiffering().size() > 10) {
+          logger.atWarning().log(
+              String.format(
+                  "Unequal reserved lists detected, Cloud SQL list with revision"
+                      + " id %d has %d different records than the current"
+                      + " Datastore list.",
+                  cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
+        } else {
+          StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
+          diff.entriesDiffering().entrySet().stream()
+              .forEach(
+                  entry -> {
+                    String label = entry.getKey();
+                    ValueDifference<ReservedListEntry> valueDiff = entry.getValue();
+                    diffMessage.append(
+                        String.format(
+                            "Domain label %s has entry %s in Datastore and entry"
+                                + " %s in Cloud SQL.\n",
+                            label, valueDiff.leftValue(), valueDiff.rightValue()));
+                  });
+          logger.atWarning().log(diffMessage.toString());
+        }
+      }
+    } else {
+      logger.atWarning().log("Reserved list in Cloud SQL is empty.");
+    }
+  }
+}

core/src/main/java/google/registry/model/registry/label/ReservedListSqlDao.java

@@ -12,20 +12,46 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.schema.tld;
+package google.registry.model.registry.label;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.util.concurrent.UncheckedExecutionException;
 import java.util.Optional;
-import java.util.concurrent.ExecutionException;
 
-/** Data access object class for {@link ReservedList} */
-public class ReservedListDao {
+/**
+ * A {@link ReservedList} DAO for Cloud SQL.
+ *
+ * <p>TODO(b/160993806): Rename this class to ReservedListDao after migrating to Cloud SQL.
+ */
+public class ReservedListSqlDao {
+
+  private ReservedListSqlDao() {}
 
   /** Persist a new reserved list to Cloud SQL. */
   public static void save(ReservedList reservedList) {
-    jpaTm().transact(() -> jpaTm().getEntityManager().persist(reservedList));
+    checkArgumentNotNull(reservedList, "Must specify reservedList");
+    jpaTm().transact(() -> jpaTm().saveNew(reservedList));
+  }
+
+  /**
+   * Returns the most recent revision of the {@link ReservedList} with the specified name, if it
+   * exists.
+   */
+  public static Optional<ReservedList> getLatestRevision(String reservedListName) {
+    return jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .getEntityManager()
+                    .createQuery(
+                        "FROM ReservedList rl LEFT JOIN FETCH rl.reservedListMap WHERE"
+                            + " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl"
+                            + " WHERE subrl.name = :name)",
+                        ReservedList.class)
+                    .setParameter("name", reservedListName)
+                    .getResultStream()
+                    .findFirst());
   }
 
   /**
@@ -46,39 +72,4 @@ public class ReservedListDao {
                         .size()
                     > 0);
   }
-
-  /**
-   * Returns the most recent revision of the {@link ReservedList} with the specified name, if it
-   * exists. TODO(shicong): Change this method to package level access after dual-read phase.
-   */
-  public static Optional<ReservedList> getLatestRevision(String reservedListName) {
-    return jpaTm()
-        .transact(
-            () ->
-                jpaTm()
-                    .getEntityManager()
-                    .createQuery(
-                        "FROM ReservedList rl LEFT JOIN FETCH rl.labelsToReservations WHERE"
-                            + " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl"
-                            + " WHERE subrl.name = :name)",
-                        ReservedList.class)
-                    .setParameter("name", reservedListName)
-                    .getResultStream()
-                    .findFirst());
-  }
-
-  /**
-   * Returns the most recent revision of the {@link ReservedList} with the specified name, from
-   * cache.
-   */
-  public static Optional<ReservedList> getLatestRevisionCached(String reservedListName) {
-    try {
-      return ReservedListCache.cacheReservedLists.get(reservedListName);
-    } catch (ExecutionException e) {
-      throw new UncheckedExecutionException(
-          "Could not retrieve reserved list named " + reservedListName, e);
-    }
-  }
-
-  private ReservedListDao() {}
 }

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -185,6 +185,10 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
   @Transient // domain-specific
   Set<DomainTransactionRecord> domainTransactionRecords;
 
+  public Long getId() {
+    return id;
+  }
+
   public Key<? extends EppResource> getParent() {
     return parent;
   }

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java

@@ -14,18 +14,20 @@
 
 package google.registry.model.reporting;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.util.CollectionUtils.isNullOrEmpty;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import google.registry.util.DomainNameUtils;
+import java.util.Set;
 import javax.persistence.Column;
 import javax.persistence.Entity;
-import javax.persistence.EnumType;
-import javax.persistence.Enumerated;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
@@ -36,11 +38,11 @@ import org.joda.time.LocalDate;
 @Entity
 @Table(
     indexes = {
-      @Index(name = "safebrowsing_threat_registrar_id_idx", columnList = "registrarId"),
-      @Index(name = "safebrowsing_threat_tld_idx", columnList = "tld"),
-      @Index(name = "safebrowsing_threat_check_date_idx", columnList = "checkDate")
+      @Index(name = "spec11threatmatch_registrar_id_idx", columnList = "registrarId"),
+      @Index(name = "spec11threatmatch_tld_idx", columnList = "tld"),
+      @Index(name = "spec11threatmatch_check_date_idx", columnList = "checkDate")
     })
-public class SafeBrowsingThreat extends ImmutableObject implements Buildable, SqlEntity {
+public class Spec11ThreatMatch extends ImmutableObject implements Buildable, SqlEntity {
 
   /** The type of threat detected. */
   public enum ThreatType {
@@ -60,10 +62,9 @@ public class SafeBrowsingThreat extends ImmutableObject implements Buildable, Sq
   @Column(nullable = false)
   String domainName;
 
-  /** The type of threat detected. */
+  /** The types of threat detected. */
   @Column(nullable = false)
-  @Enumerated(EnumType.STRING)
-  ThreatType threatType;
+  Set<ThreatType> threatTypes;
 
   /** Primary key of the domain table and unique identifier for all EPP resources. */
   @Column(nullable = false)
@@ -89,8 +90,8 @@ public class SafeBrowsingThreat extends ImmutableObject implements Buildable, Sq
     return domainName;
   }
 
-  public ThreatType getThreatType() {
-    return threatType;
+  public ImmutableSet<ThreatType> getThreatTypes() {
+    return ImmutableSet.copyOf(threatTypes);
   }
 
   public String getDomainRepoId() {
@@ -119,18 +120,18 @@ public class SafeBrowsingThreat extends ImmutableObject implements Buildable, Sq
     return new Builder(clone(this));
   }
 
-  /** A builder for constructing {@link SafeBrowsingThreat}, since it is immutable. */
-  public static class Builder extends Buildable.Builder<SafeBrowsingThreat> {
+  /** A builder for constructing {@link Spec11ThreatMatch}, since it is immutable. */
+  public static class Builder extends Buildable.Builder<Spec11ThreatMatch> {
     public Builder() {}
 
-    private Builder(SafeBrowsingThreat instance) {
+    private Builder(Spec11ThreatMatch instance) {
       super(instance);
     }
 
     @Override
-    public SafeBrowsingThreat build() {
+    public Spec11ThreatMatch build() {
       checkArgumentNotNull(getInstance().domainName, "Domain name cannot be null");
-      checkArgumentNotNull(getInstance().threatType, "Threat type cannot be null");
+      checkArgumentNotNull(getInstance().threatTypes, "Threat types cannot be null");
       checkArgumentNotNull(getInstance().domainRepoId, "Repo ID cannot be null");
       checkArgumentNotNull(getInstance().registrarId, "Registrar ID cannot be null");
       checkArgumentNotNull(getInstance().checkDate, "Check date cannot be null");
@@ -145,8 +146,10 @@ public class SafeBrowsingThreat extends ImmutableObject implements Buildable, Sq
       return this;
     }
 
-    public Builder setThreatType(ThreatType threatType) {
-      getInstance().threatType = threatType;
+    public Builder setThreatTypes(ImmutableSet<ThreatType> threatTypes) {
+      checkArgument(!isNullOrEmpty(threatTypes), "threatTypes cannot be null or empty.");
+
+      getInstance().threatTypes = threatTypes;
       return this;
     }
 

core/src/main/java/google/registry/persistence/JpaRetries.java

@@ -0,0 +1,58 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
+import java.sql.SQLException;
+import java.util.function.Predicate;
+import javax.persistence.OptimisticLockException;
+
+/** Helpers for identifying retriable database operations. */
+public final class JpaRetries {
+
+  private JpaRetries() {}
+
+  private static final ImmutableSet<String> RETRIABLE_TXN_SQL_STATE =
+      ImmutableSet.of(
+          "40001", // serialization_failure
+          "40P01", // deadlock_detected, PSQL-specific
+          "55006", // object_in_use, PSQL and DB2
+          "55P03" // lock_not_available, PSQL-specific
+          );
+
+  private static final Predicate<Throwable> RETRIABLE_TXN_PREDICATE =
+      Predicates.or(
+          OptimisticLockException.class::isInstance,
+          e ->
+              e instanceof SQLException
+                  && RETRIABLE_TXN_SQL_STATE.contains(((SQLException) e).getSQLState()));
+
+  public static boolean isFailedTxnRetriable(Throwable throwable) {
+    Throwable t = throwable;
+    while (t != null) {
+      if (RETRIABLE_TXN_PREDICATE.test(t)) {
+        return true;
+      }
+      t = t.getCause();
+    }
+    return false;
+  }
+
+  public static boolean isFailedQueryRetriable(Throwable throwable) {
+    // TODO(weiminyu): check for more error codes.
+    return isFailedTxnRetriable(throwable);
+  }
+}

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -94,6 +94,21 @@ public class PersistenceModule {
       @Config("cloudSqlJdbcUrl") String jdbcUrl,
       @Config("cloudSqlInstanceConnectionName") String instanceConnectionName,
       @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs) {
+    return createPartialSqlConfigs(jdbcUrl, instanceConnectionName, defaultConfigs);
+  }
+
+  @Provides
+  @Singleton
+  @BeamPipelineCloudSqlConfigs
+  static ImmutableMap<String, String> provideBeamPipelineCloudSqlConfigs(
+      @Config("beamCloudSqlJdbcUrl") String jdbcUrl,
+      @Config("beamCloudSqlInstanceConnectionName") String instanceConnectionName,
+      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs) {
+    return createPartialSqlConfigs(jdbcUrl, instanceConnectionName, defaultConfigs);
+  }
+
+  private static ImmutableMap<String, String> createPartialSqlConfigs(
+      String jdbcUrl, String instanceConnectionName, ImmutableMap<String, String> defaultConfigs) {
     HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
     overrides.put(Environment.URL, jdbcUrl);
     overrides.put(HIKARI_DS_SOCKET_FACTORY, "com.google.cloud.sql.postgres.SocketFactory");
@@ -135,13 +150,15 @@ public class PersistenceModule {
   @Singleton
   @SocketFactoryJpaTm
   static JpaTransactionManager provideSocketFactoryJpaTm(
-      @Config("cloudSqlUsername") String username,
-      @Config("cloudSqlPassword") String password,
-      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
+      @Config("beamCloudSqlUsername") String username,
+      @Config("beamCloudSqlPassword") String password,
+      @Config("beamHibernateHikariMaximumPoolSize") int hikariMaximumPoolSize,
+      @BeamPipelineCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, password);
+    overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(hikariMaximumPoolSize));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -149,9 +166,9 @@ public class PersistenceModule {
   @Singleton
   @JdbcJpaTm
   static JpaTransactionManager provideLocalJpaTm(
-      @Config("cloudSqlJdbcUrl") String jdbcUrl,
-      @Config("cloudSqlUsername") String username,
-      @Config("cloudSqlPassword") String password,
+      @Config("beamCloudSqlJdbcUrl") String jdbcUrl,
+      @Config("beamCloudSqlUsername") String username,
+      @Config("beamCloudSqlPassword") String password,
       @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs,
       Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
@@ -218,6 +235,11 @@ public class PersistenceModule {
   @Documented
   @interface PartialCloudSqlConfigs {}
 
+  /** Dagger qualifier for the Cloud SQL configs used by Beam pipelines. */
+  @Qualifier
+  @Documented
+  @interface BeamPipelineCloudSqlConfigs {}
+
   /** Dagger qualifier for the default Hibernate configurations. */
   // TODO(shicong): Change annotations in this class to none public or put them in a top level
   //  package

core/src/main/java/google/registry/persistence/VKey.java

@@ -58,6 +58,31 @@ public class VKey<T> extends ImmutableObject implements Serializable {
     return new VKey(kind, null, sqlKey);
   }
 
+  /** Creates a {@link VKey} which only contains the ofy primary key. */
+  public static <T> VKey<T> createOfy(
+      Class<? extends T> kind, com.googlecode.objectify.Key<? extends T> ofyKey) {
+    checkArgumentNotNull(kind, "kind must not be null");
+    checkArgumentNotNull(ofyKey, "ofyKey must not be null");
+    return new VKey(kind, ofyKey, null);
+  }
+
+  /**
+   * Creates a {@link VKey} which only contains the ofy primary key by specifying the id of the
+   * {@link Key}.
+   */
+  public static <T> VKey<T> createOfy(Class<? extends T> kind, long id) {
+    return createOfy(kind, Key.create(kind, id));
+  }
+
+  /**
+   * Creates a {@link VKey} which only contains the ofy primary key by specifying the name of the
+   * {@link Key}.
+   */
+  public static <T> VKey<T> createOfy(Class<? extends T> kind, String name) {
+    checkArgumentNotNull(kind, "name must not be null");
+    return createOfy(kind, Key.create(kind, name));
+  }
+
   /** Creates a {@link VKey} which only contains both sql and ofy primary key. */
   public static <T> VKey<T> create(
       Class<? extends T> kind, Object sqlKey, com.googlecode.objectify.Key ofyKey) {

core/src/main/java/google/registry/persistence/converter/Spec11ThreatMatchThreatTypeSetConverter.java

@@ -0,0 +1,34 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA {@link AttributeConverter} for storing/retrieving {@code Set}. */
+@Converter(autoApply = true)
+public class Spec11ThreatMatchThreatTypeSetConverter extends StringSetConverterBase<ThreatType> {
+
+  @Override
+  String toString(ThreatType element) {
+    return element.name();
+  }
+
+  @Override
+  ThreatType fromString(String value) {
+    return ThreatType.valueOf(value);
+  }
+}

core/src/main/java/google/registry/persistence/converter/StringMapConverterBase.java

@@ -14,7 +14,7 @@
 
 package google.registry.persistence.converter;
 
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 
 import google.registry.persistence.converter.StringMapDescriptor.StringMap;
 import java.util.Map;
@@ -38,7 +38,7 @@ public abstract class StringMapConverterBase<K, V>
         : StringMap.create(
             attribute.entrySet().stream()
                 .map(this::convertToDatabaseMapEntry)
-                .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
+                .collect(entriesToImmutableMap()));
   }
 
   @Override
@@ -47,6 +47,6 @@ public abstract class StringMapConverterBase<K, V>
         ? null
         : dbData.getMap().entrySet().stream()
             .map(this::convertToEntityMapEntry)
-            .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+            .collect(entriesToImmutableMap());
   }
 }

core/src/main/java/google/registry/persistence/converter/TimedTransitionPropertyConverterBase.java

@@ -14,8 +14,9 @@
 
 package google.registry.persistence.converter;
 
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
@@ -45,7 +46,7 @@ public abstract class TimedTransitionPropertyConverterBase<K, V extends TimedTra
         : StringMap.create(
             attribute.entrySet().stream()
                 .map(this::convertToDatabaseMapEntry)
-                .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
+                .collect(entriesToImmutableMap()));
   }
 
   @Override
@@ -53,10 +54,10 @@ public abstract class TimedTransitionPropertyConverterBase<K, V extends TimedTra
     if (dbData == null) {
       return null;
     }
-    Map<DateTime, K> map =
+    ImmutableMap<DateTime, K> map =
         dbData.getMap().entrySet().stream()
             .map(this::convertToEntityMapEntry)
-            .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+            .collect(entriesToImmutableMap());
     return TimedTransitionProperty.fromValueMap(
         ImmutableSortedMap.copyOf(map), getTimedTransitionSubclass());
   }

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManager.java

@@ -25,4 +25,11 @@ public interface JpaTransactionManager extends TransactionManager {
 
   /** Deletes the entity by its id, throws exception if the entity is not deleted. */
   public abstract <T> void assertDelete(VKey<T> key);
+
+  /**
+   * Releases all resources and shuts down.
+   *
+   * <p>The errorprone check forbids injection of {@link java.io.Closeable} resources.
+   */
+  void teardown();
 }

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -15,18 +15,21 @@
 package google.registry.persistence.transaction;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+import static java.util.AbstractMap.SimpleEntry;
 import static java.util.stream.Collectors.joining;
 
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import google.registry.persistence.VKey;
 import google.registry.util.Clock;
 import java.lang.reflect.Field;
+import java.util.Map.Entry;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
@@ -59,6 +62,11 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     this.clock = clock;
   }
 
+  @Override
+  public void teardown() {
+    emf.close();
+  }
+
   @Override
   public EntityManager getEntityManager() {
     if (transactionInfo.get().entityManager == null) {
@@ -98,7 +106,8 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
       T result = work.get();
       txn.commit();
       return result;
-    } catch (RuntimeException e) {
+    } catch (RuntimeException | Error e) {
+      // Error is unchecked!
       try {
         txn.rollback();
         logger.atWarning().log("Error during transaction; transaction rolled back");
@@ -253,20 +262,18 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public <T> ImmutableList<T> load(Iterable<VKey<T>> keys) {
+  public <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys) {
     checkArgumentNotNull(keys, "keys must be specified");
     assertInTransaction();
     return StreamSupport.stream(keys.spliterator(), false)
+        // Accept duplicate keys.
+        .distinct()
         .map(
-            key -> {
-              T entity = getEntityManager().find(key.getKind(), key.getSqlKey());
-              if (entity == null) {
-                throw new NoSuchElementException(
-                    key.getKind().getName() + " with key " + key.getSqlKey() + " not found.");
-              }
-              return entity;
-            })
-        .collect(toImmutableList());
+            key ->
+                new SimpleEntry<VKey<? extends T>, T>(
+                    key, getEntityManager().find(key.getKind(), key.getSqlKey())))
+        .filter(entry -> entry.getValue() != null)
+        .collect(toImmutableMap(Entry::getKey, Entry::getValue));
   }
 
   @Override

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -16,6 +16,7 @@ package google.registry.persistence.transaction;
 
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import google.registry.persistence.VKey;
 import java.util.NoSuchElementException;
 import java.util.Optional;
@@ -119,7 +120,7 @@ public interface TransactionManager {
    *
    * @throws NoSuchElementException if any of the keys are not found.
    */
-  <T> ImmutableList<T> load(Iterable<VKey<T>> keys);
+  <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys);
 
   /** Loads all entities of the given type, returns empty if there is no such entity. */
   <T> ImmutableList<T> loadAll(Class<T> clazz);

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -75,7 +75,12 @@ public class TransactionManagerFactory {
     return tm;
   }
 
-  /** Returns {@link JpaTransactionManager} instance. */
+  /**
+   * Returns {@link JpaTransactionManager} instance.
+   *
+   * <p>Between invocations of {@link TransactionManagerFactory#setJpaTm} every call to this method
+   * returns the same instance.
+   */
   public static JpaTransactionManager jpaTm() {
     return jpaTm.get();
   }
@@ -93,7 +98,7 @@ public class TransactionManagerFactory {
         RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
             || RegistryToolEnvironment.get() != null,
         "setJpamTm() should only be called by tools and tests.");
-    jpaTm = jpaTmSupplier;
+    jpaTm = Suppliers.memoize(jpaTmSupplier::get);
   }
 
   /** Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}. */

core/src/main/java/google/registry/pricing/PricingEngineProxy.java

@@ -22,7 +22,6 @@ import google.registry.model.pricing.PremiumPricingEngine;
 import google.registry.model.pricing.PremiumPricingEngine.DomainPrices;
 import google.registry.model.registry.Registry;
 import java.util.Map;
-import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
@@ -52,11 +51,6 @@ public final class PricingEngineProxy {
     return getPricesForDomainName(domainName, priceTime).isPremium();
   }
 
-  /** Returns the fee class of the specified domain name. */
-  public static Optional<String> getDomainFeeClass(String domainName, DateTime priceTime) {
-    return getPricesForDomainName(domainName, priceTime).getFeeClass();
-  }
-
   /**
    * Returns the full {@link DomainPrices} details for the given domain name by dispatching to the
    * appropriate {@link PremiumPricingEngine} based on what is configured for the TLD that the

core/src/main/java/google/registry/rdap/RdapJsonFormatter.java

@@ -342,7 +342,7 @@ public class RdapJsonFormatter {
     // Kick off the database loads of the nameservers that we will need, so it can load
     // asynchronously while we load and process the contacts.
     ImmutableSet<HostResource> loadedHosts =
-        ImmutableSet.copyOf(tm().load(domainBase.getNameservers()));
+        ImmutableSet.copyOf(tm().load(domainBase.getNameservers()).values());
     // Load the registrant and other contacts and add them to the data.
     Map<Key<ContactResource>, ContactResource> loadedContacts =
         ofy()

core/src/main/java/google/registry/rde/RdeModule.java

@@ -89,7 +89,7 @@ public abstract class RdeModule {
   @Provides
   @Parameter(PARAM_LENIENT)
   static boolean provideLenient(HttpServletRequest req) {
-    return extractBooleanParameter(req, PARAM_REVISION);
+    return extractBooleanParameter(req, PARAM_LENIENT);
   }
 
   @Provides

core/src/main/java/google/registry/rde/RdeResourceType.java

@@ -29,7 +29,7 @@ public enum RdeResourceType {
   DOMAIN("urn:ietf:params:xml:ns:rdeDomain-1.0", EnumSet.of(FULL, THIN)),
   HOST("urn:ietf:params:xml:ns:rdeHost-1.0", EnumSet.of(FULL)),
   REGISTRAR("urn:ietf:params:xml:ns:rdeRegistrar-1.0", EnumSet.of(FULL, THIN)),
-  IDN("urn:ietf:params:xml:ns:rdeIDN-1.0", EnumSet.of(FULL, THIN)),
+  IDN("urn:ietf:params:xml:ns:rdeIDN-1.0", EnumSet.of(FULL)),
   HEADER("urn:ietf:params:xml:ns:rdeHeader-1.0", EnumSet.of(FULL, THIN));
 
   private final String uri;

core/src/main/java/google/registry/rde/RdeStagingMapper.java

@@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSetMultimap;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.googlecode.objectify.Result;
 import google.registry.model.EppResource;
 import google.registry.model.contact.ContactResource;
@@ -44,6 +45,11 @@ public final class RdeStagingMapper extends Mapper<EppResource, PendingDeposit,
 
   private static final long serialVersionUID = -1518185703789372524L;
 
+  // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
+  // if sneaks into production we would get an extra signal.
+  private static final ImmutableSet<Registrar.Type> IGNORED_REGISTRAR_TYPES =
+      Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
+
   private final RdeMarshaller marshaller;
   private final ImmutableSetMultimap<String, PendingDeposit> pendings;
 
@@ -64,6 +70,9 @@ public final class RdeStagingMapper extends Mapper<EppResource, PendingDeposit,
     if (resource == null) {
       long registrarsEmitted = 0;
       for (Registrar registrar : Registrar.loadAllCached()) {
+        if (IGNORED_REGISTRAR_TYPES.contains(registrar.getType())) {
+          continue;
+        }
         DepositFragment fragment = marshaller.marshalRegistrar(registrar);
         for (PendingDeposit pending : pendings.values()) {
           emit(pending, fragment);

core/src/main/java/google/registry/rde/RdeStagingReducer.java

@@ -77,7 +77,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
   private final byte[] stagingKeyBytes;
   private final RdeMarshaller marshaller;
 
-  private RdeStagingReducer(
+  RdeStagingReducer(
       TaskQueueUtils taskQueueUtils,
       LockHandler lockHandler,
       int gcsBufferSize,
@@ -125,7 +125,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
     final DateTime watermark = key.watermark();
     final int revision =
         Optional.ofNullable(key.revision())
-            .orElse(RdeRevision.getNextRevision(tld, watermark, mode));
+            .orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
     String id = RdeUtil.timestampToId(watermark);
     String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
     if (key.manual()) {
@@ -168,9 +168,13 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
           logger.atSevere().log("Fragment error: %s", fragment.error());
         }
       }
-      for (IdnTableEnum idn : IdnTableEnum.values()) {
-        output.write(marshaller.marshalIdn(idn.getTable()));
-        counter.increment(RdeResourceType.IDN);
+
+      // Don't write the IDN elements for BRDA.
+      if (mode == RdeMode.FULL) {
+        for (IdnTableEnum idn : IdnTableEnum.values()) {
+          output.write(marshaller.marshalIdn(idn.getTable()));
+          counter.increment(RdeResourceType.IDN);
+        }
       }
 
       // Output XML that says how many resources were emitted.

core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java

@@ -15,7 +15,6 @@
 package google.registry.reporting.icann;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -27,6 +26,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteStreams;
 import com.googlecode.objectify.Key;
@@ -209,9 +209,9 @@ public final class IcannReportingUploadAction implements Runnable {
 
     ImmutableSet<Registry> registries = Registries.getTldEntitiesOfType(TldType.REAL);
 
-    Map<Key<Cursor>, Registry> activityKeyMap =
+    ImmutableMap<Key<Cursor>, Registry> activityKeyMap =
         loadKeyMap(registries, CursorType.ICANN_UPLOAD_ACTIVITY);
-    Map<Key<Cursor>, Registry> transactionKeyMap =
+    ImmutableMap<Key<Cursor>, Registry> transactionKeyMap =
         loadKeyMap(registries, CursorType.ICANN_UPLOAD_TX);
 
     ImmutableSet.Builder<Key<Cursor>> keys = new ImmutableSet.Builder<>();
@@ -229,9 +229,9 @@ public final class IcannReportingUploadAction implements Runnable {
     return cursors.build();
   }
 
-  private Map<Key<Cursor>, Registry> loadKeyMap(
+  private ImmutableMap<Key<Cursor>, Registry> loadKeyMap(
       ImmutableSet<Registry> registries, CursorType type) {
-    return registries.stream().collect(toImmutableMap(r -> Cursor.createKey(type, r), r -> r));
+    return Maps.uniqueIndex(registries, r -> Cursor.createKey(type, r));
   }
 
   /**

core/src/main/java/google/registry/schema/cursor/CursorDao.java

@@ -15,7 +15,6 @@
 package google.registry.schema.cursor;
 
 import static com.google.appengine.api.search.checkers.Preconditions.checkNotNull;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -23,6 +22,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
 import com.google.common.flogger.FluentLogger;
 import google.registry.model.common.Cursor.CursorType;
 import google.registry.schema.cursor.Cursor.CursorId;
@@ -171,11 +171,11 @@ public class CursorDao {
       // Load all the cursors of that type from Cloud SQL
       List<Cursor> cloudSqlCursors = loadByType(type);
 
-      // Create a map of each tld to its cursor if one exists
+      // Create a map of each TLD to its cursor if one exists.
       ImmutableMap<String, Cursor> cloudSqlCursorMap =
-          cloudSqlCursors.stream().collect(toImmutableMap(c -> c.getScope(), c -> c));
+          Maps.uniqueIndex(cloudSqlCursors, Cursor::getScope);
 
-      // Compare each Datastore cursor with its corresponding Cloud SQL cursor
+      // Compare each Datastore cursor with its corresponding Cloud SQL cursor.
       for (google.registry.model.common.Cursor cursor : cursors.keySet()) {
         Cursor cloudSqlCursor = cloudSqlCursorMap.get(cursors.get(cursor));
         compare(cursor, cloudSqlCursor, cursors.get(cursor));