Remove extra space after bashslash (#743 )

Add retrier to JpaTransactionManager (#693 )
* Add retrier to retry exceptions we've seen so far * Check for nested exceptions * fix formatting * check for nested transactions * add JDBCConnectionException to isFailedQueryRetriable * Add retrier to methods with suppliers
2026-07-10 10:02:50 +00:00 · 2020-08-04 20:00:46 -04:00 · 2020-08-04 12:22:08 -04:00 · 2020-08-04 11:06:02 -04:00 · 2020-08-03 17:40:50 -04:00 · 2020-08-03 14:40:00 -07:00
956 changed files with 26167 additions and 17452 deletions
@@ -79,6 +79,10 @@ nomulus.iml
 nomulus.ipr
 nomulus.iws

+# Auto-generated java classes by Intellij
+*/src/main/generated/
+*/src/test/generated_tests/
+
 # VScode
 .vscode

@@ -93,8 +97,9 @@ nomulus.iws
 ######################################################################
 # Gradle Ignores

-# We don't want to ignore the gradle jar files
+# We don't want to ignore the gradle or google-java-format jar files
 !/gradle/wrapper/**/*.jar
+!/java-format/*.jar
 .gradle/
 **/build
 cloudbuild-caches/
@@ -2,3 +2,5 @@ extraction:
  java:
    prepare:
      packages: "npm"
+    index:
+      java_version: "11"
@@ -25,7 +25,7 @@ buildscript {

  dependencies {
    classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.0.1'
-    classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.6.1"
+    classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.6.1'
    classpath 'org.sonatype.aether:aether-api:1.13.1'
    classpath 'org.sonatype.aether:aether-impl:1.13.1'
  }
@@ -35,9 +35,6 @@ plugins {
  // Java static analysis plugins. Keep versions consistent with
  // ./buildSrc/build.gradle
  id 'nebula.lint' version '16.0.2'
-  // TODO(weiminyu): consider remove net.ltgt.apt. Gradle 5.2+
-  //  has similar functionalities.
-  id 'net.ltgt.apt' version '0.19' apply false
  id 'net.ltgt.errorprone' version '0.6.1'
  id 'checkstyle'
  id 'com.github.johnrengelman.shadow' version '5.1.0'
@@ -49,6 +46,7 @@ plugins {
  id 'com.diffplug.gradle.spotless' version '3.25.0'

  id 'jacoco'
+  id 'com.dorongold.task-tree' version '1.5'
 }

 wrapper {
@@ -289,12 +287,6 @@ subprojects {

  project.tasks.test.dependsOn runPresubmits

-// Path to code generated with annotation processors. Note that this path is
-// chosen by the 'net.ltgt.apt' plugin, and may change if IDE-specific plugins
-// are applied, e.g., 'idea' or 'eclipse'
-  def aptGeneratedDir = "${project.buildDir}/generated/source/apt/main"
-  def aptGeneratedTestDir = "${project.buildDir}/generated/source/apt/test"
-
  def commonlyExcludedResources = ['**/*.java', '**/BUILD']

  project.ext.javaDir = "${project.projectDir}/src/main/java"
@@ -303,18 +295,12 @@ subprojects {

  sourceSets {
    main {
-      java {
-        srcDirs += aptGeneratedDir
-      }
      resources {
        srcDirs += project.ext.javaDir
        exclude commonlyExcludedResources
      }
    }
    test {
-      java {
-        srcDirs += aptGeneratedTestDir
-      }
      resources {
        srcDirs += project.ext.javaTestDir
        exclude commonlyExcludedResources
@@ -322,9 +308,14 @@ subprojects {
    }
  }

-  javadocSource << project.sourceSets.main.allJava
-  javadocClasspath << project.sourceSets.main.compileClasspath
-  javadocDependentTasks << project.tasks.compileJava
+  // No need to produce javadoc for the docs subproject, which has no APIs to
+  // expose to users.
+  if (project.name != 'docs') {
+    javadocSource << project.sourceSets.main.allJava
+    javadocClasspath << project.sourceSets.main.compileClasspath
+    javadocClasspath << "${buildDir}/generated/sources/annotationProcessor/java/main"
+    javadocDependentTasks << project.tasks.compileJava
+  }
 }

 // If "-P verboseTestOutput=true" is passed in, configure all subprojects to dump all of their
@@ -443,6 +434,7 @@ task javaIncrementalFormatDryRun {
    println("${invokeJavaDiffFormatScript("show")}")
  }
 }
+tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)

 // Checks if modified lines in Java source files need reformatting.
 // Note that this task processes modified Java files in the entire repository.
@@ -452,8 +444,6 @@ task javaIncrementalFormatApply {
  }
 }

-tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)
-
 task javadoc(type: Javadoc) {
  source javadocSource
  classpath = files(javadocClasspath)
@@ -468,4 +458,16 @@ task javadoc(type: Javadoc) {

 tasks.build.dependsOn(tasks.javadoc)

+// Task for doing development on core Nomulus.
+// This fixes code formatting automatically as necessary, builds and tests the
+// core Nomulus codebase, and runs all presubmits.
+task coreDev {
+  dependsOn 'javaIncrementalFormatApply'
+  dependsOn 'javadoc'
+  dependsOn 'checkDependenciesDotGradle'
+  dependsOn 'checkLicense'
+  dependsOn ':core:check'
+  dependsOn 'assemble'
+}
+
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
@@ -26,9 +26,6 @@ buildscript {
 plugins {
  // Java static analysis plugins. Keep versions consistent with ../build.gradle
  id 'nebula.lint' version '16.0.2'
-  // Config helper for annotation processors such as AutoValue and Dagger.
-  // Ensures that source code is generated at an appropriate location.
-  id 'net.ltgt.apt' version '0.19' apply false
  id 'net.ltgt.errorprone' version '0.6.1'
  id 'checkstyle'
  id 'com.diffplug.gradle.spotless' version '3.25.0'
@@ -83,10 +80,8 @@ dependencies {
  annotationProcessor deps['com.google.auto.value:auto-value']
  testCompile deps['com.google.truth:truth']
  testCompile deps['com.google.truth.extensions:truth-java8-extension']
-  testCompile deps['junit:junit']
  testCompile deps['org.junit.jupiter:junit-jupiter-api']
  testCompile deps['org.junit.jupiter:junit-jupiter-engine']
-  testCompile deps['org.junit.vintage:junit-vintage-engine']
  testCompile deps['org.mockito:mockito-core']
 }

@@ -19,9 +19,7 @@ commons-lang:commons-lang:2.6
 javax.inject:javax.inject:1
 junit:junit:4.12
 nebula.lint:nebula.lint.gradle.plugin:16.0.2
-net.ltgt.apt:net.ltgt.apt.gradle.plugin:0.19
 net.ltgt.errorprone:net.ltgt.errorprone.gradle.plugin:0.6.1
-net.ltgt.gradle:gradle-apt-plugin:0.19
 net.ltgt.gradle:gradle-errorprone-plugin:0.6.1
 org.apache.commons:commons-lang3:3.8.1
 org.apache.maven:maven-artifact:3.6.2
@@ -1,4 +0,0 @@
-# This is a Gradle generated file for dependency locking.
-# Manual edits can break the build and are not advised.
-# This file is expected to be part of source control.
-com.google.errorprone:javac:9+181-r4173-1
@@ -49,9 +49,9 @@ javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
-junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
-net.bytebuddy:byte-buddy:1.9.7
+junit:junit:4.12
+net.bytebuddy:byte-buddy-agent:1.10.5
+net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.8
@@ -61,13 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
-org.mockito:mockito-core:2.25.0
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
+org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm-analysis:6.0
@@ -49,9 +49,9 @@ javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
-junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
-net.bytebuddy:byte-buddy:1.9.7
+junit:junit:4.12
+net.bytebuddy:byte-buddy-agent:1.10.5
+net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.8
@@ -61,13 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
-org.mockito:mockito-core:2.25.0
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
+org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm-analysis:6.0
@@ -49,9 +49,9 @@ javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
-junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
-net.bytebuddy:byte-buddy:1.9.7
+junit:junit:4.12
+net.bytebuddy:byte-buddy-agent:1.10.5
+net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.8
@@ -61,13 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
-org.mockito:mockito-core:2.25.0
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
+org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm-analysis:6.0
@@ -49,9 +49,9 @@ javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
-junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
-net.bytebuddy:byte-buddy:1.9.7
+junit:junit:4.12
+net.bytebuddy:byte-buddy-agent:1.10.5
+net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.8
@@ -61,13 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
-org.mockito:mockito-core:2.25.0
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
+org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm-analysis:6.0
@@ -29,6 +29,7 @@ import google.registry.gradle.plugin.ProjectData.TaskData;
 import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Map;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;

@@ -162,7 +163,7 @@ final class CoverPageGenerator {
            task.reports().entrySet().stream()
                .collect(
                    toImmutableMap(
-                        entry -> entry.getKey(),
+                        Map.Entry::getKey,
                        entry ->
                            entry.getValue().files().isEmpty()
                                ? ""
@@ -57,6 +57,10 @@ final class GcsPluginUtils {
    return file.toPath().toAbsolutePath().normalize();
  }

+  static Path toNormalizedPath(Path file) {
+    return file.toAbsolutePath().normalize();
+  }
+
  static String getContentType(String fileName) {
    return EXTENSION_TO_CONTENT_TYPE.getOrDefault(
        Files.getFileExtension(fileName), DEFAULT_CONTENT_TYPE);
@@ -168,7 +172,7 @@ final class GcsPluginUtils {
            .collect(
                toImmutableMap(
                    file -> rootDir.relativize(toNormalizedPath(file)),
-                    file -> toByteArraySupplier(file)));
+                    GcsPluginUtils::toByteArraySupplier));

    if (files.isEmpty()) {
      // The directory exists, but is empty. Return empty FilesWithEntryPoint
@@ -26,13 +26,10 @@ import com.google.common.collect.ImmutableSet;
 import google.registry.gradle.plugin.ProjectData.TaskData;
 import java.io.File;
 import java.nio.file.Paths;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import org.junit.jupiter.api.Test;

 /** Tests for {@link CoverPageGenerator} */
-@RunWith(JUnit4.class)
-public final class CoverPageGeneratorTest {
+final class CoverPageGeneratorTest {

  private static final ProjectData EMPTY_PROJECT =
      ProjectData.builder()
@@ -88,14 +85,14 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testGetFilesToUpload_entryPoint_isIndexHtml() {
+  void testGetFilesToUpload_entryPoint_isIndexHtml() {
    CoverPageGenerator coverPageGenerator = new CoverPageGenerator(EMPTY_PROJECT);
    assertThat(coverPageGenerator.getFilesToUpload().entryPoint())
        .isEqualTo(Paths.get("index.html"));
  }

  @Test
-  public void testGetFilesToUpload_containsEntryFile() {
+  void testGetFilesToUpload_containsEntryFile() {
    String content = getContentOfGeneratedFile(EMPTY_PROJECT, "index.html");
    assertThat(content)
        .contains(
@@ -103,7 +100,7 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCoverPage_showsFailedTask() {
+  void testCoverPage_showsFailedTask() {
    String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_FAILURE).build());
    assertThat(content).contains("task-failure");
    assertThat(content).contains("<p>FAILURE</p>");
@@ -112,7 +109,7 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCoverPage_showsSuccessfulTask() {
+  void testCoverPage_showsSuccessfulTask() {
    String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_SUCCESS).build());
    assertThat(content).contains("task-success");
    assertThat(content).doesNotContain("<p>FAILURE</p>");
@@ -121,7 +118,7 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCoverPage_showsUpToDateTask() {
+  void testCoverPage_showsUpToDateTask() {
    String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_UP_TO_DATE).build());
    assertThat(content).contains("task-up-to-date");
    assertThat(content).doesNotContain("<p>FAILURE</p>");
@@ -130,11 +127,10 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCoverPage_failedAreFirst() {
+  void testCoverPage_failedAreFirst() {
    String content =
        getCoverPage(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                .addTask(EMPTY_TASK_UP_TO_DATE)
                .addTask(EMPTY_TASK_FAILURE)
                .addTask(EMPTY_TASK_SUCCESS)
@@ -149,11 +145,10 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCoverPage_failingTask_statusIsFailure() {
+  void testCoverPage_failingTask_statusIsFailure() {
    String content =
        getCoverPage(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                .addTask(EMPTY_TASK_UP_TO_DATE)
                .addTask(EMPTY_TASK_FAILURE)
                .addTask(EMPTY_TASK_SUCCESS)
@@ -162,11 +157,10 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCoverPage_noFailingTask_statusIsSuccess() {
+  void testCoverPage_noFailingTask_statusIsSuccess() {
    String content =
        getCoverPage(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                .addTask(EMPTY_TASK_UP_TO_DATE)
                .addTask(EMPTY_TASK_SUCCESS)
                .build());
@@ -174,7 +168,7 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testGetFilesToUpload_containsCssFile() {
+  void testGetFilesToUpload_containsCssFile() {
    ImmutableMap<String, String> files = getGeneratedFiles(EMPTY_PROJECT);
    assertThat(files).containsKey(filenameJoiner.join("css", "style.css"));
    assertThat(files.get(filenameJoiner.join("css", "style.css"))).contains("body {");
@@ -183,14 +177,12 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCreateReportFiles_taskWithLog() {
+  void testCreateReportFiles_taskWithLog() {
    ImmutableMap<String, String> files =
        getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                .addTask(
-                    EMPTY_TASK_SUCCESS
-                        .toBuilder()
+                    EMPTY_TASK_SUCCESS.toBuilder()
                        .setUniqueName("my:name")
                        .setLog(toByteArraySupplier("my log data"))
                        .build())
@@ -200,11 +192,10 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCreateReportFiles_taskWithoutLog() {
+  void testCreateReportFiles_taskWithoutLog() {
    ImmutableMap<String, String> files =
        getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                .addTask(EMPTY_TASK_SUCCESS.toBuilder().setUniqueName("my:name").build())
                .build());
    assertThat(files).doesNotContainKey("logs/my-name.log");
@@ -212,14 +203,12 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCreateReportFiles_taskWithFilledReport() {
+  void testCreateReportFiles_taskWithFilledReport() {
    ImmutableMap<String, String> files =
        getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                .addTask(
-                    EMPTY_TASK_SUCCESS
-                        .toBuilder()
+                    EMPTY_TASK_SUCCESS.toBuilder()
                        .putReport(
                            "someReport",
                            FilesWithEntryPoint.create(
@@ -234,14 +223,12 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCreateReportFiles_taskWithEmptyReport() {
+  void testCreateReportFiles_taskWithEmptyReport() {
    ImmutableMap<String, String> files =
        getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                .addTask(
-                    EMPTY_TASK_SUCCESS
-                        .toBuilder()
+                    EMPTY_TASK_SUCCESS.toBuilder()
                        .putReport(
                            "someReport",
                            FilesWithEntryPoint.create(
@@ -254,14 +241,12 @@ public final class CoverPageGeneratorTest {
  }

  @Test
-  public void testCreateReportFiles_taskWithLogAndMultipleReports() {
+  void testCreateReportFiles_taskWithLogAndMultipleReports() {
    ImmutableMap<String, String> files =
        getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                .addTask(
-                    EMPTY_TASK_SUCCESS
-                        .toBuilder()
+                    EMPTY_TASK_SUCCESS.toBuilder()
                        .setUniqueName("my:name")
                        .setLog(toByteArraySupplier("log data"))
                        .putReport(
@@ -23,6 +23,9 @@ import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath;
 import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs;
 import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.createDirectories;
+import static java.nio.file.Files.createDirectory;
+import static java.nio.file.Files.createFile;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -36,22 +39,20 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Optional;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;

 /** Tests for {@link GcsPluginUtilsTest} */
-@RunWith(JUnit4.class)
-public final class GcsPluginUtilsTest {
+final class GcsPluginUtilsTest {

  private static final Joiner filenameJoiner = Joiner.on(File.separator);

-  @Rule public final TemporaryFolder folder = new TemporaryFolder();
+  @SuppressWarnings("WeakerAccess")
+  @TempDir
+  Path tmpDir;

  @Test
-  public void testGetContentType_knownTypes() {
+  void testGetContentType_knownTypes() {
    assertThat(getContentType("path/to/file.html")).isEqualTo("text/html");
    assertThat(getContentType("path/to/file.htm")).isEqualTo("text/html");
    assertThat(getContentType("path/to/file.log")).isEqualTo("text/plain");
@@ -63,12 +64,12 @@ public final class GcsPluginUtilsTest {
  }

  @Test
-  public void testGetContentType_unknownTypes() {
+  void testGetContentType_unknownTypes() {
    assertThat(getContentType("path/to/file.unknown")).isEqualTo("application/octet-stream");
  }

  @Test
-  public void testUploadFileToGcs() {
+  void testUploadFileToGcs() {
    Storage storage = mock(Storage.class);
    uploadFileToGcs(
        storage, "my-bucket", Paths.get("my", "filename.txt"), toByteArraySupplier("my data"));
@@ -82,7 +83,7 @@ public final class GcsPluginUtilsTest {
  }

  @Test
-  public void testUploadFilesToGcsMultithread() {
+  void testUploadFilesToGcsMultithread() {
    Storage storage = mock(Storage.class);
    uploadFilesToGcsMultithread(
        storage,
@@ -121,21 +122,21 @@ public final class GcsPluginUtilsTest {
  }

  @Test
-  public void testToByteArraySupplier_string() {
+  void testToByteArraySupplier_string() {
    assertThat(toByteArraySupplier("my string").get()).isEqualTo("my string".getBytes(UTF_8));
  }

  @Test
-  public void testToByteArraySupplier_stringSupplier() {
+  void testToByteArraySupplier_stringSupplier() {
    assertThat(toByteArraySupplier(() -> "my string").get()).isEqualTo("my string".getBytes(UTF_8));
  }

  @Test
-  public void testToByteArraySupplier_file() throws Exception {
-    folder.newFolder("arbitrary");
-    File file = folder.newFile("arbitrary/file.txt");
-    Files.write(file.toPath(), "some data".getBytes(UTF_8));
-    assertThat(toByteArraySupplier(file).get()).isEqualTo("some data".getBytes(UTF_8));
+  void testToByteArraySupplier_file() throws Exception {
+    Path dir = createDirectory(tmpDir.resolve("arbitrary"));
+    Path file = createFile(dir.resolve("file.txt"));
+    Files.write(file, "some data".getBytes(UTF_8));
+    assertThat(toByteArraySupplier(file.toFile()).get()).isEqualTo("some data".getBytes(UTF_8));
  }

  private ImmutableMap<String, String> readAllFiles(FilesWithEntryPoint reportFiles) {
@@ -147,16 +148,16 @@ public final class GcsPluginUtilsTest {
  }

  @Test
-  public void testCreateReportFiles_destinationIsFile() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    folder.newFolder("my", "root", "some", "path");
-    File destination = folder.newFile("my/root/some/path/file.txt");
-    Files.write(destination.toPath(), "some data".getBytes(UTF_8));
+  void testCreateReportFiles_destinationIsFile() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path somePath = createDirectories(root.resolve("some/path"));
+    Path destination = createFile(somePath.resolve("file.txt"));
+    Files.write(destination, "some data".getBytes(UTF_8));
    // Since the entry point is obvious here - any hint given is just ignored.
-    File ignoredHint = folder.newFile("my/root/ignored.txt");
+    File ignoredHint = createFile(root.resolve("ignored.txt")).toFile();

    FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root);

    assertThat(files.entryPoint().toString())
        .isEqualTo(filenameJoiner.join("some", "path", "file.txt"));
@@ -165,13 +166,13 @@ public final class GcsPluginUtilsTest {
  }

  @Test
-  public void testCreateReportFiles_destinationDoesntExist() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
+  void testCreateReportFiles_destinationDoesntExist() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
    File destination = root.resolve("non/existing.txt").toFile();
    assertThat(destination.isFile()).isFalse();
    assertThat(destination.isDirectory()).isFalse();
-    // Since there are not files, any hint given is obvioulsy wrong and will be ignored.
-    File ignoredHint = folder.newFile("my/root/ignored.txt");
+    // Since there are no files, any hint given is obviously wrong and will be ignored.
+    File ignoredHint = createFile(root.resolve("ignored.txt")).toFile();

    FilesWithEntryPoint files =
        readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
@@ -181,34 +182,33 @@ public final class GcsPluginUtilsTest {
  }

  @Test
-  public void testCreateReportFiles_noFiles() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
-    folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
-    // Since there are not files, any hint given is obvioulsy wrong and will be ignored.
-    File ignoredHint = folder.newFile("my/root/ignored.txt");
+  void testCreateReportFiles_noFiles() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
+    createDirectories(destination.resolve("a/b"));
+    createDirectory(destination.resolve("c"));
+    // Since there are not files, any hint given is obviously wrong and will be ignored.
+    File ignoredHint = createFile(root.resolve("ignored.txt")).toFile();

    FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root);

    assertThat(files.entryPoint().toString()).isEqualTo(filenameJoiner.join("some", "path"));
    assertThat(files.files()).isEmpty();
  }

  @Test
-  public void testCreateReportFiles_oneFile() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
-    folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
-    Files.write(
-        folder.newFile("my/root/some/path/a/file.txt").toPath(), "some data".getBytes(UTF_8));
+  void testCreateReportFiles_oneFile() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
+    createDirectories(destination.resolve("a/b"));
+    createDirectory(destination.resolve("c"));
+    Files.write(createFile(destination.resolve("a/file.txt")), "some data".getBytes(UTF_8));
    // Since the entry point is obvious here - any hint given is just ignored.
-    File ignoredHint = folder.newFile("my/root/ignored.txt");
+    File ignoredHint = createFile(root.resolve("ignored.txt")).toFile();

    FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root);

    assertThat(files.entryPoint().toString())
        .isEqualTo(filenameJoiner.join("some", "path", "a", "file.txt"));
@@ -222,22 +222,19 @@ public final class GcsPluginUtilsTest {
   * <p>TODO(guyben): switch to checking zip file instead.
   */
  @Test
-  public void testCreateReportFiles_multipleFiles_noHint() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
-    folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
+  void testCreateReportFiles_multipleFiles_noHint() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
+    createDirectories(destination.resolve("a/b"));
+    createDirectory(destination.resolve("c"));

-    Files.write(
-        folder.newFile("my/root/some/path/index.html").toPath(), "some data".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("index.html")), "some data".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8));

-    FilesWithEntryPoint files = readFilesWithEntryPoint(destination, Optional.empty(), root);
+    FilesWithEntryPoint files =
+        readFilesWithEntryPoint(destination.toFile(), Optional.empty(), root);

    assertThat(files.entryPoint().toString())
        .isEqualTo(filenameJoiner.join("some", "path", "path.zip"));
@@ -251,24 +248,20 @@ public final class GcsPluginUtilsTest {
   * <p>TODO(guyben): switch to checking zip file instead.
   */
  @Test
-  public void testCreateReportFiles_multipleFiles_withBadHint() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
+  void testCreateReportFiles_multipleFiles_withBadHint() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
    // This entry point points to a directory, which isn't an appropriate entry point
-    File badEntryPoint = folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
+    File badEntryPoint = createDirectories(destination.resolve("a/b")).toFile();
+    createDirectory(destination.resolve("c"));

-    Files.write(
-        folder.newFile("my/root/some/path/index.html").toPath(), "some data".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("index.html")), "some data".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8));

    FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(badEntryPoint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(badEntryPoint), root);

    assertThat(files.entryPoint().toString())
        .isEqualTo(filenameJoiner.join("some", "path", "path.zip"));
@@ -277,24 +270,21 @@ public final class GcsPluginUtilsTest {
  }

  @Test
-  public void testCreateReportFiles_multipleFiles_withGoodHint() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
-    folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
+  void testCreateReportFiles_multipleFiles_withGoodHint() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
+    createDirectories(destination.resolve("a/b"));
+    createDirectory(destination.resolve("c"));
    // The hint is an actual file nested in the destination directory!
-    File goodEntryPoint = folder.newFile("my/root/some/path/index.html");
+    Path goodEntryPoint = createFile(destination.resolve("index.html"));

-    Files.write(goodEntryPoint.toPath(), "some data".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
+    Files.write(goodEntryPoint, "some data".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8));

    FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(goodEntryPoint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(goodEntryPoint.toFile()), root);

    assertThat(files.entryPoint().toString())
        .isEqualTo(filenameJoiner.join("some", "path", "index.html"));
@@ -61,8 +61,6 @@ dependencies {
  testingCompile deps['com.google.truth:truth']
  testingCompile deps['io.github.java-diff-utils:java-diff-utils']

-  testCompile deps['junit:junit']
  testCompile deps['org.junit.jupiter:junit-jupiter-api']
  testCompile deps['org.junit.jupiter:junit-jupiter-engine']
-  testCompile deps['org.junit.vintage:junit-vintage-engine']
 }
@@ -1,4 +1,3 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-com.google.errorprone:javac:9+181-r4173-1
@@ -14,16 +14,15 @@ com.googlecode.java-diff-utils:diffutils:1.3.0
 io.github.java-diff-utils:java-diff-utils:4.0
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 org.apiguardian:apiguardian-api:1.1.0
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0
@@ -14,16 +14,15 @@ com.googlecode.java-diff-utils:diffutils:1.3.0
 io.github.java-diff-utils:java-diff-utils:4.0
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 org.apiguardian:apiguardian-api:1.1.0
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0
@@ -15,16 +15,15 @@ com.googlecode.java-diff-utils:diffutils:1.3.0
 io.github.java-diff-utils:java-diff-utils:4.0
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 org.apiguardian:apiguardian-api:1.1.0
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0
@@ -15,16 +15,15 @@ com.googlecode.java-diff-utils:diffutils:1.3.0
 io.github.java-diff-utils:java-diff-utils:4.0
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 org.apiguardian:apiguardian-api:1.1.0
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0
@@ -17,7 +17,7 @@ package google.registry.testing.truth;
 import static com.google.common.io.Resources.getResource;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.testing.truth.TextDiffSubject.assertThat;
-import static org.junit.Assert.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertThrows;

 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
@@ -25,13 +25,10 @@ import com.google.common.io.Resources;
 import google.registry.testing.truth.TextDiffSubject.DiffFormat;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import org.junit.jupiter.api.Test;

 /** Unit tests for {@link TextDiffSubject}. */
-@RunWith(JUnit4.class)
-public class TextDiffSubjectTest {
+class TextDiffSubjectTest {

  private static final String RESOURCE_FOLDER = "google/registry/testing/truth/";
  // Resources for input data.
@@ -44,21 +41,21 @@ public class TextDiffSubjectTest {
      RESOURCE_FOLDER + "text-sidebyside-diff.txt";

  @Test
-  public void unifiedDiff_equal() throws IOException {
+  void unifiedDiff_equal() throws IOException {
    assertThat(getResource(ACTUAL_RESOURCE))
        .withDiffFormat(DiffFormat.UNIFIED_DIFF)
        .hasSameContentAs(getResource(ACTUAL_RESOURCE));
  }

  @Test
-  public void sideBySideDiff_equal() throws IOException {
+  void sideBySideDiff_equal() throws IOException {
    assertThat(getResource(ACTUAL_RESOURCE))
        .withDiffFormat(DiffFormat.SIDE_BY_SIDE_MARKDOWN)
        .hasSameContentAs(getResource(ACTUAL_RESOURCE));
  }

  @Test
-  public void unifedDiff_notEqual() throws IOException {
+  void unifedDiff_notEqual() throws IOException {
    assertThrows(
        AssertionError.class,
        () ->
@@ -68,7 +65,7 @@ public class TextDiffSubjectTest {
  }

  @Test
-  public void sideBySideDiff_notEqual() throws IOException {
+  void sideBySideDiff_notEqual() throws IOException {
    assertThrows(
        AssertionError.class,
        () ->
@@ -78,13 +75,13 @@ public class TextDiffSubjectTest {
  }

  @Test
-  public void displayed_unifiedDiff_noDiff() throws IOException {
+  void displayed_unifiedDiff_noDiff() throws IOException {
    ImmutableList<String> actual = readAllLinesFromResource(ACTUAL_RESOURCE);
    assertThat(TextDiffSubject.generateUnifiedDiff(actual, actual)).isEqualTo("");
  }

  @Test
-  public void displayed_unifiedDiff_hasDiff() throws IOException {
+  void displayed_unifiedDiff_hasDiff() throws IOException {
    ImmutableList<String> actual = readAllLinesFromResource(ACTUAL_RESOURCE);
    ImmutableList<String> expected = readAllLinesFromResource(EXPECTED_RESOURCE);
    String diff = Joiner.on('\n').join(readAllLinesFromResource(UNIFIED_DIFF_RESOURCE));
@@ -92,7 +89,7 @@ public class TextDiffSubjectTest {
  }

  @Test
-  public void displayed_sideBySideDiff_hasDiff() throws IOException {
+  void displayed_sideBySideDiff_hasDiff() throws IOException {
    ImmutableList<String> actual = readAllLinesFromResource(ACTUAL_RESOURCE);
    ImmutableList<String> expected = readAllLinesFromResource(EXPECTED_RESOURCE);
    String diff = Joiner.on('\n').join(readAllLinesFromResource(SIDE_BY_SIDE_DIFF_RESOURCE));
@@ -16,6 +16,7 @@ package google.registry.testing.truth;

 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.truth.Truth.assertAbout;
+import static com.google.common.truth.Truth.assertWithMessage;
 import static java.nio.charset.StandardCharsets.UTF_8;

 import com.github.difflib.DiffUtils;
@@ -31,6 +32,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.io.Resources;
 import com.google.common.truth.Fact;
 import com.google.common.truth.FailureMetadata;
+import com.google.common.truth.SimpleSubjectBuilder;
 import com.google.common.truth.Subject;
 import java.io.IOException;
 import java.net.URL;
@@ -68,6 +70,15 @@ public class TextDiffSubject extends Subject {
    this.actual = ImmutableList.copyOf(actual);
  }

+  protected TextDiffSubject(FailureMetadata metadata, URL actual) {
+    super(metadata, actual);
+    try {
+      this.actual = ImmutableList.copyOf(Resources.asCharSource(actual, UTF_8).readLines());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
  public TextDiffSubject withDiffFormat(DiffFormat format) {
    this.diffFormat = format;
    return this;
@@ -100,6 +111,11 @@ public class TextDiffSubject extends Subject {
    return assertThat(Resources.asCharSource(resourceUrl, UTF_8).readLines());
  }

+  public static SimpleSubjectBuilder<TextDiffSubject, URL> assertWithMessageAboutUrlSource(
+      String format, Object... params) {
+    return assertWithMessage(format, params).about(urlFactory());
+  }
+
  private static final Subject.Factory<TextDiffSubject, ImmutableList<String>>
      TEXT_DIFF_SUBJECT_TEXT_FACTORY = TextDiffSubject::new;

@@ -107,6 +123,13 @@ public class TextDiffSubject extends Subject {
    return TEXT_DIFF_SUBJECT_TEXT_FACTORY;
  }

+  private static final Subject.Factory<TextDiffSubject, URL> TEXT_DIFF_SUBJECT_URL_FACTORY =
+      TextDiffSubject::new;
+
+  public static Subject.Factory<TextDiffSubject, URL> urlFactory() {
+    return TEXT_DIFF_SUBJECT_URL_FACTORY;
+  }
+
  static String generateUnifiedDiff(
      ImmutableList<String> expectedContent, ImmutableList<String> actualContent) {
    Patch<String> diff;
@@ -19,6 +19,7 @@ import argparse
 import attr
 import io
 import os
+import shutil
 import subprocess
 import sys
 from typing import List, Union
@@ -49,15 +50,30 @@ PROPERTIES_HEADER = """\
 # This file defines properties used by the gradle build.  It must be kept in
 # sync with config/nom_build.py.
 #
-# To regenerate, run config/nom_build.py --generate-gradle-properties
+# To regenerate, run ./nom_build --generate-gradle-properties
 #
 # To view property descriptions (which are command line flags for
-# nom_build), run config/nom_build.py --help.
+# nom_build), run ./nom_build --help.
 #
 # DO NOT EDIT THIS FILE BY HAND
 org.gradle.jvmargs=-Xmx1024m
 """

+# Help text to be displayed (in addition to the synopsis and flag help, which
+# are displayed automatically).
+HELP_TEXT = """\
+A wrapper around the gradle build that provides the following features:
+
+-   Converts properties into flags to guard against property name spelling errors
+    and to provide help descriptions for all properties.
+-   Provides pseudo-commands (with the ":nom:" prefix) that encapsulate common
+    actions that are difficult to implement in gradle.
+
+Pseudo-commands:
+    :nom:generate_golden_file - regenerates the golden file from the current
+        set of flyway files.
+"""
+
 # Define all of our special gradle properties here.
 PROPERTIES = [
    Property('mavenUrl',
@@ -114,6 +130,11 @@ PROPERTIES = [
    Property('nomulus_version',
             'The version of nomulus to test against in a database '
             'integration test.'),
+    Property('dot_path',
+             'The path to "dot", part of the graphviz package that converts '
+             'a BEAM pipeline to image. Setting this property to empty string '
+             'will disable image generation.',
+             '/usr/bin/dot'),
 ]

 GRADLE_FLAGS = [
@@ -251,8 +272,42 @@ def get_root() -> str:
    return cur_dir


-def main(args):
-    parser = argparse.ArgumentParser('nom_build')
+class Abort(Exception):
+    """Raised to terminate the process with a non-zero error code.
+
+    Parameters are ignored.
+    """
+
+
+def do_pseudo_task(task: str) -> None:
+    root = get_root()
+    if task == ':nom:generate_golden_file':
+        if not subprocess.call([f'{root}/gradlew', ':db:test']):
+            print('\033[33mWARNING:\033[0m Golden schema appears to be '
+                  'up-to-date.  If you are making schema changes,  be sure to '
+                  'add a flyway file for them.')
+            return
+        print('\033[33mWARNING:\033[0m Ignore the above failure, it is '
+              'expected.')
+
+        # Copy the new schema into place.
+        shutil.copy(f'{root}/db/build/resources/test/testcontainer/'
+                    'mount/dump.txt',
+                    f'{root}/db/src/main/resources/sql/schema/'
+                    'nomulus.golden.sql')
+
+        if subprocess.call([f'{root}/gradlew', ':db:test']):
+            print('\033[31mERROR:\033[0m Golden file test failed after '
+                  'copying schema.  Please check your flyway files.')
+            raise Abort()
+    else:
+        print(f'\033[31mERROR:\033[0m Unknown task {task}')
+        raise Abort()
+
+
+def main(args) -> int:
+    parser = argparse.ArgumentParser('nom_build', description=HELP_TEXT,
+                                     formatter_class=argparse.RawTextHelpFormatter)
    for prop in PROPERTIES:
        parser.add_argument('--' + prop.name, default=prop.default,
                            help=prop.desc)
@@ -291,7 +346,7 @@ def main(args):
    if args.generate_gradle_properties:
        with open(f'{root}/gradle.properties', 'w') as dst:
            dst.write(gradle_properties)
-        return
+        return 0

    # Verify that the gradle properties file is what we expect it to be.
    with open(f'{root}/gradle.properties') as src:
@@ -316,12 +371,39 @@ def main(args):
            if flag.has_arg:
                gradle_command.append(arg_val)

+    # See if there are any special ":nom:" pseudo-tasks specified.
+    got_non_pseudo_tasks = False
+    for arg in args.non_flag_args[1:]:
+        if arg.startswith(':nom:'):
+            if got_non_pseudo_tasks:
+                # We can't currently deal with the situation of gradle tasks
+                # before pseudo-tasks.  This could be implemented by invoking
+                # gradle for only the set of gradle tasks before the pseudo
+                # task, but that's overkill for now.
+                print(f'\033[31mERROR:\033[0m Pseudo task ({arg}) must be '
+                      'specified prior to all actual gradle tasks.  Aborting.')
+                return 1
+            do_pseudo_task(arg)
+        else:
+            got_non_pseudo_tasks = True
+    non_flag_args = [
+        arg for arg in args.non_flag_args[1:] if not arg.startswith(':nom:')]
+
+    if not non_flag_args:
+        if not got_non_pseudo_tasks:
+            print('\033[33mWARNING:\033[0m No tasks specified.  Not '
+                  'doing anything')
+        return 0
+
    # Add the non-flag args (we exclude the first, which is the command name
    # itself) and run.
-    gradle_command.extend(args.non_flag_args[1:])
-    subprocess.call(gradle_command)
+    gradle_command.extend(non_flag_args)
+    return subprocess.call(gradle_command)


 if __name__ == '__main__':
-    main(sys.argv)
+    try:
+        sys.exit(main(sys.argv))
+    except Abort as ex:
+        sys.exit(1)

@@ -14,6 +14,7 @@

 import io
 import os
+import shutil
 import unittest
 from unittest import mock
 import nom_build
@@ -67,6 +68,7 @@ class MyTest(unittest.TestCase):
            mock.patch.object(nom_build, 'print', self.print_fake).start())

        self.call_mock = mock.patch.object(subprocess, 'call').start()
+        self.copy_mock = mock.patch.object(shutil, 'copy').start()

        self.file_contents = {
            # Prefil with the actual file contents.
@@ -92,17 +94,32 @@ class MyTest(unittest.TestCase):

    def test_no_args(self):
        nom_build.main(['nom_build'])
-        self.assertEqual(self.printed, [])
-        self.call_mock.assert_called_with([GRADLEW])
+        self.assertEqual(self.printed,
+                         ['\x1b[33mWARNING:\x1b[0m No tasks specified.  Not '
+                         'doing anything'])

    def test_property_calls(self):
-        nom_build.main(['nom_build', '--testFilter=foo'])
-        self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo'])
+        nom_build.main(['nom_build', 'task-name', '--testFilter=foo'])
+        self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo',
+                                          'task-name'])

    def test_gradle_flags(self):
-        nom_build.main(['nom_build', '-d', '-b', 'foo'])
+        nom_build.main(['nom_build', 'task-name', '-d', '-b', 'foo'])
        self.call_mock.assert_called_with([GRADLEW, '--build-file', 'foo',
-                                          '--debug'])
+                                          '--debug', 'task-name'])
+
+    def test_generate_golden_file(self):
+        self.call_mock.side_effect = [1, 0]
+        nom_build.main(['nom_build', ':nom:generate_golden_file'])
+        self.call_mock.assert_has_calls([
+            mock.call([GRADLEW, ':db:test']),
+            mock.call([GRADLEW, ':db:test'])
+        ])
+
+    def test_generate_golden_file_nofail(self):
+        self.call_mock.return_value = 0
+        nom_build.main(['nom_build', ':nom:generate_golden_file'])
+        self.call_mock.assert_has_calls([mock.call([GRADLEW, ':db:test'])])

 unittest.main()

@@ -22,7 +22,7 @@ import sys
 import re

 # We should never analyze any generated files
-UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/"}
+UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/"}
 # We can't rely on CI to have the Enum package installed so we do this instead.
 FORBIDDEN = 1
 REQUIRED = 2
@@ -77,9 +77,9 @@ PRESUBMITS = {
    PresubmitCheck(
        r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
        ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
-            ".git", "/build/", "/generated/", "node_modules/",
-            "JUnitBackports.java", "registrar_bin.", "registrar_dbg.",
-            "google-java-format-diff.py",
+            ".git", "/build/", "/generated/", "/generated_tests/",
+            "node_modules/", "JUnitBackports.java", "registrar_bin.",
+            "registrar_dbg.", "google-java-format-diff.py",
            "nomulus.golden.sql", "soyutils_usegoog.js"
        }, REQUIRED):
        "File did not include the license header.",
@@ -93,20 +93,20 @@ PRESUBMITS = {
    PresubmitCheck(
        r".*\bSystem\.(out|err)\.print", "java", {
            "StackdriverDashboardBuilder.java", "/tools/", "/example/",
-            "RegistryTestServerMain.java", "TestServerRule.java",
+            "RegistryTestServerMain.java", "TestServerExtension.java",
            "FlowDocumentationTool.java"
        }):
        "System.(out|err).println is only allowed in tools/ packages. Please "
        "use a logger instead.",

-    # ObjectifyService.register is restricted to main/ or AppEngineRule.
+    # ObjectifyService.register is restricted to main/ or AppEngineExtension.
    PresubmitCheck(
        r".*\bObjectifyService\.register", "java", {
            "/build/", "/generated/", "node_modules/", "src/main/",
-            "AppEngineRule.java"
+            "AppEngineExtension.java"
        }):
-      "ObjectifyService.register is not allowed in tests. Please use "
-      "AppengineRule.register instead.",
+      "ObjectifyService.register(...) is not allowed in tests. Please use "
+      "AppEngineExtension.register(...) instead.",

    # PostgreSQLContainer instantiation must specify docker tag
    PresubmitCheck(
@@ -22,7 +22,7 @@ plugins {

 // Path to code generated by ad hoc tasks in this project. A separate path is
 // used for easy inspection.
-def generatedDir = "${project.buildDir}/generated/source/custom/main"
+def generatedDir = "${project.buildDir}/generated/sources/custom/java/main"
 def resourcesDir = "${project.buildDir}/resources/main"
 def screenshotsDir = "${project.buildDir}/screenshots"
 def screenshotsForGoldensDir = "${project.buildDir}/screenshots_for_goldens"
@@ -82,11 +82,6 @@ sourceSets {
  main {
    java {
      srcDirs += generatedDir
-      // Javadoc API is deprecated in Java 11 and removed in Java 12.
-      // TODO(jianglai): re-enable after migrating to the new Javadoc API
-      if ((JavaVersion.current().majorVersion as Integer) >= 11) {
-        exclude 'google/registry/documentation/**'
-      }
    }
    resources {
      exclude '**/*.xjb'
@@ -238,6 +233,7 @@ dependencies {
  compile deps['jline:jline']
  compile deps['joda-time:joda-time']
  compile deps['org.apache.avro:avro']
+  testCompile deps['org.apache.beam:beam-runners-core-construction-java']
  testCompile deps['org.apache.beam:beam-runners-direct-java']
  compile deps['org.apache.beam:beam-runners-google-cloud-dataflow-java']
  compile deps['org.apache.beam:beam-sdks-java-core']
@@ -256,12 +252,13 @@ dependencies {
  compile deps['org.bouncycastle:bcpg-jdk15on']
  testCompile deps['org.bouncycastle:bcpkix-jdk15on']
  compile deps['org.bouncycastle:bcprov-jdk15on']
+  testCompile deps['com.fasterxml.jackson.core:jackson-databind']
  runtime deps['org.glassfish.jaxb:jaxb-runtime']
  compile deps['org.hibernate:hibernate-core']
  compile deps['org.joda:joda-money']
  compile deps['org.json:json']
  testCompile deps['org.mortbay.jetty:jetty']
-  runtimeOnly deps['org.postgresql:postgresql']
+  compile deps['org.postgresql:postgresql']
  testCompile deps['org.seleniumhq.selenium:selenium-api']
  testCompile deps['org.seleniumhq.selenium:selenium-chrome-driver']
  testCompile deps['org.seleniumhq.selenium:selenium-java']
@@ -272,7 +269,6 @@ dependencies {
  compile deps['org.testcontainers:postgresql']
  testCompile deps['org.testcontainers:selenium']
  testCompile deps['org.testcontainers:testcontainers']
-  testCompile deps['pl.pragmatists:JUnitParams']
  compile deps['xerces:xmlParserAPIs']
  compile deps['xpp3:xpp3']
  // This dependency must come after javax.mail:mail as it would otherwise
@@ -312,10 +308,12 @@ dependencies {
  testCompile deps['org.junit.jupiter:junit-jupiter-api']
  testCompile deps['org.junit.jupiter:junit-jupiter-engine']
  testCompile deps['org.junit.jupiter:junit-jupiter-migrationsupport']
+  testCompile deps['org.junit.jupiter:junit-jupiter-params']
+  testCompile deps['org.junit-pioneer:junit-pioneer']
  testCompile deps['org.junit.platform:junit-platform-runner']
  testCompile deps['org.junit.platform:junit-platform-suite-api']
-  testCompile deps['org.junit.vintage:junit-vintage-engine']
  testCompile deps['org.mockito:mockito-core']
+  testCompile deps['org.mockito:mockito-junit-jupiter']
  runtime deps['org.postgresql:postgresql']

  // Indirect dependency found by undeclared-dependency check. Such
@@ -406,6 +404,9 @@ task jaxbToJava {
        }
      }
    }
+    execInBash(
+        'find . -name *.java -exec sed -i /\\*\\ \\<p\\>\\$/d {} +',
+        generatedDir)
  }
 }

@@ -704,6 +705,10 @@ task fragileTest(type: FilteringTest) {

  // Run every test class in a freshly started process.
  forkEvery 1
+
+  doFirst {
+    new File(screenshotsDir).deleteDir()
+  }
 }

 task outcastTest(type: FilteringTest) {
@@ -821,6 +826,30 @@ createToolTask(
    'google.registry.tools.DevTool',
    sourceSets.nonprod)

+project.tasks.create('initSqlPipeline', JavaExec) {
+  main = 'google.registry.beam.initsql.InitSqlPipeline'
+
+  doFirst {
+    getToolArgsList().ifPresent {
+      args it
+    }
+
+    def isDirectRunner =
+        args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
+    // The dependency containing DirectRunner is intentionally excluded from the
+    // production binary, so that it won't be chosen by mistake: we definitely do
+    // not want to use it for the real jobs, yet DirectRunner is the default if
+    // the user forgets to override it.
+    // DirectRunner is required for tests and is already on testRuntimeClasspath.
+    // For simplicity, we add testRuntimeClasspath to this task's classpath instead
+    // of defining a new configuration just for the DirectRunner dependency.
+    classpath =
+        isDirectRunner
+            ? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
+            : sourceSets.main.runtimeClasspath
+  }
+}
+
 project.tasks.create('generateSqlSchema', JavaExec) {
  classpath = sourceSets.nonprod.runtimeClasspath
  main = 'google.registry.tools.DevTool'
@@ -848,22 +877,6 @@ task generateGoldenImages(type: FilteringTest) {
 }
 generateGoldenImages.finalizedBy(findGoldenImages)

-task flowDocsTool(type: JavaExec) {
-  systemProperty 'test.projectRoot', rootProject.projectRootDir
-  systemProperty 'test.resourcesDir', resourcesDir
-
-  classpath = sourceSets.main.runtimeClasspath
-  main = 'google.registry.documentation.FlowDocumentationTool'
-
-  def arguments = []
-  if (rootProject.flowDocsFile) {
-    arguments << "--output_file=${rootProject.flowDocsFile}"
-  } else {
-    arguments << "--output_file=${rootProject.projectRootDir}/docs/flows.md"
-  }
-  args arguments
-}
-
 task standardTest(type: FilteringTest) {
  includeAllTests()
  exclude fragileTestPatterns
@@ -888,10 +901,6 @@ task standardTest(type: FilteringTest) {

  systemProperty 'test.projectRoot', rootProject.projectRootDir
  systemProperty 'test.resourcesDir', resourcesDir
-
-  doFirst {
-    new File(screenshotsDir).deleteDir()
-  }
 }

 test {
@@ -965,6 +974,49 @@ task buildToolImage(dependsOn: nomulus, type: Exec) {
  commandLine 'docker', 'build', '-t', 'nomulus-tool', '.'
 }

+task generateInitSqlPipelineGraph(type: Test) {
+  include "**/InitSqlPipelineGraphTest.*"
+  testNameIncludePatterns = ["**createPipeline_compareGraph"]
+  ignoreFailures = true
+}
+
+task updateInitSqlPipelineGraph(type: Copy) {
+  def graphRelativePath = 'google/registry/beam/initsql/'
+  from ("${projectDir}/build/resources/test/${graphRelativePath}") {
+    include 'pipeline_curr.dot'
+    rename 'curr', 'golden'
+  }
+  into "src/test/resources/${graphRelativePath}"
+
+  dependsOn generateInitSqlPipelineGraph
+
+  doLast {
+    if (com.google.common.base.Strings.isNullOrEmpty(project.dot_path)) {
+      getLogger().info('Property dot_path is null. Not creating image for pipeline graph.')
+    }
+    def dotPath = project.dot_path
+    if (!new File(dotPath).exists()) {
+      throw new RuntimeException(
+          """\
+              ${dotPath} not found. Make sure graphviz is installed
+              and the dot_path property is set correctly."""
+              .stripIndent())
+    }
+    def goldenGraph = "src/test/resources/${graphRelativePath}/pipeline_golden.dot"
+    def goldenImage = "src/test/resources/${graphRelativePath}/pipeline_golden.png"
+    def cmd = "${dotPath} -Tpng -o \"${goldenImage}\" \"${goldenGraph}\""
+    try {
+      rootProject.ext.execInBash(cmd, projectDir)
+    } catch (Throwable throwable) {
+      throw new RuntimeException(
+          """\
+              Failed to generate golden image with command ${cmd}
+              Error: ${throwable.getMessage()}
+          """)
+    }
+  }
+}
+
 // Build the devtool jar.
 createUberJar(
    'devtool',
@@ -228,6 +228,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
@@ -223,6 +223,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
@@ -1,4 +1,3 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-com.google.errorprone:javac:9+181-r4173-1
@@ -228,6 +228,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
@@ -226,6 +226,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
@@ -1,3 +0,0 @@
-# This is a Gradle generated file for dependency locking.
-# Manual edits can break the build and are not advised.
-# This file is expected to be part of source control.
@@ -186,7 +186,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.3
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -243,18 +243,20 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
 org.junit.platform:junit-platform-runner:1.6.2
 org.junit.platform:junit-platform-suite-api:1.6.2
-org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
-org.mockito:mockito-core:2.25.0
+org.mockito:mockito-core:3.3.3
+org.mockito:mockito-junit-jupiter:3.3.3
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
 org.objenesis:objenesis:2.6
@@ -264,6 +266,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
@@ -290,6 +293,5 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
-pl.pragmatists:JUnitParams:1.1.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c
@@ -184,7 +184,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.3
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -241,18 +241,20 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
 org.junit.platform:junit-platform-runner:1.6.2
 org.junit.platform:junit-platform-suite-api:1.6.2
-org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
-org.mockito:mockito-core:2.25.0
+org.mockito:mockito-core:3.3.3
+org.mockito:mockito-junit-jupiter:3.3.3
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
 org.objenesis:objenesis:2.6
@@ -262,6 +264,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
@@ -288,6 +291,5 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
-pl.pragmatists:JUnitParams:1.1.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c
@@ -188,7 +188,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.3
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -246,18 +246,20 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
 org.junit.platform:junit-platform-runner:1.6.2
 org.junit.platform:junit-platform-suite-api:1.6.2
-org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
-org.mockito:mockito-core:2.25.0
+org.mockito:mockito-core:3.3.3
+org.mockito:mockito-junit-jupiter:3.3.3
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
 org.objenesis:objenesis:2.6
@@ -294,6 +296,5 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
-pl.pragmatists:JUnitParams:1.1.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c
@@ -188,7 +188,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.3
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -246,18 +246,20 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
 org.junit.platform:junit-platform-runner:1.6.2
 org.junit.platform:junit-platform-suite-api:1.6.2
-org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
-org.mockito:mockito-core:2.25.0
+org.mockito:mockito-core:3.3.3
+org.mockito:mockito-junit-jupiter:3.3.3
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
 org.objenesis:objenesis:2.6
@@ -295,6 +297,5 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
-pl.pragmatists:JUnitParams:1.1.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c
@@ -39,7 +39,7 @@ module.exports = function(config) {
        included: false
      },
      {
-        pattern: 'core/build/generated/source/custom/main/**/*.soy.js',
+        pattern: 'core/build/generated/sources/custom/java/main/**/*.soy.js',
        included: false
      },
      {
@@ -65,7 +65,7 @@ module.exports = function(config) {
      'node_modules/google-closure-library/closure/goog/base.js': ['closure'],
      'node_modules/google-closure-library/closure/**/*.js': ['closure'],
      'core/src/*/javascript/**/*.js': ['closure'],
-      'core/build/generated/source/custom/main/**/*.soy.js': ['closure'],
+      'core/build/generated/sources/custom/java/main/**/*.soy.js': ['closure'],
    },
    proxies: {
      "/assets/": "/base/core/build/resources/main/google/registry/ui/assets/"
@@ -22,20 +22,35 @@ import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;

 /**
- * Sets up a placeholder {@link Environment} on a non-AppEngine platform so that Datastore Entities
- * can be deserialized. See {@code DatastoreEntityExtension} in test source for more information.
+ * Sets up a fake {@link Environment} so that the following operations can be performed without the
+ * Datastore service:
+ *
+ * <ul>
+ *   <li>Create Objectify {@code Keys}.
+ *   <li>Instantiate Objectify objects.
+ *   <li>Convert Datastore {@code Entities} to their corresponding Objectify objects.
+ * </ul>
+ *
+ * <p>User has the option to specify their desired {@code appId} string, which forms part of an
+ * Objectify {@code Key} and is included in the equality check. This feature makes it easy to
+ * compare a migrated object in SQL with the original in Objectify.
+ *
+ * <p>Note that conversion from Objectify objects to Datastore {@code Entities} still requires the
+ * Datastore service.
 */
 public class AppEngineEnvironment implements Closeable {

-  private static final Environment PLACEHOLDER_ENV = createAppEngineEnvironment();
-
  private boolean isPlaceHolderNeeded;

  public AppEngineEnvironment() {
+    this("PlaceholderAppId");
+  }
+
+  public AppEngineEnvironment(String appId) {
    isPlaceHolderNeeded = ApiProxy.getCurrentEnvironment() == null;
    // isPlaceHolderNeeded may be true when we are invoked in a test with AppEngineRule.
    if (isPlaceHolderNeeded) {
-      ApiProxy.setEnvironmentForCurrentThread(PLACEHOLDER_ENV);
+      ApiProxy.setEnvironmentForCurrentThread(createAppEngineEnvironment(appId));
    }
  }

@@ -47,7 +62,7 @@ public class AppEngineEnvironment implements Closeable {
  }

  /** Returns a placeholder {@link Environment} that can return hardcoded AppId and Attributes. */
-  private static Environment createAppEngineEnvironment() {
+  private static Environment createAppEngineEnvironment(String appId) {
    return (Environment)
        Proxy.newProxyInstance(
            Environment.class.getClassLoader(),
@@ -55,7 +70,7 @@ public class AppEngineEnvironment implements Closeable {
            (Object proxy, Method method, Object[] args) -> {
              switch (method.getName()) {
                case "getAppId":
-                  return "PlaceholderAppId";
+                  return appId;
                case "getAttributes":
                  return ImmutableMap.<String, Object>of();
                default:
@@ -169,10 +169,12 @@ public final class AsyncTaskEnqueuer {
        lock.getRelockDuration().isPresent(),
        "Lock with ID %s not configured for relock",
        lock.getRevisionId());
+    String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
    addTaskToQueueWithRetry(
        asyncActionsPushQueue,
        TaskOptions.Builder.withUrl(RelockDomainAction.PATH)
            .method(Method.POST)
+            .header("Host", backendHostname)
            .param(
                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
                String.valueOf(lock.getRevisionId()))
@@ -285,11 +285,9 @@ public class DeleteContactsAndHostsAction implements Runnable {
      if (resourceKey.getKind().equals(KIND_CONTACT)) {
        return domain
            .getReferencedContacts()
-            .contains(VKey.createOfy(ContactResource.class, (Key<ContactResource>) resourceKey));
+            .contains(VKey.from((Key<ContactResource>) resourceKey));
      } else if (resourceKey.getKind().equals(KIND_HOST)) {
-        return domain
-            .getNameservers()
-            .contains(VKey.createOfy(HostResource.class, (Key<HostResource>) resourceKey));
+        return domain.getNameservers().contains(VKey.from((Key<HostResource>) resourceKey));
      } else {
        throw new IllegalStateException("EPP resource key of unknown type: " + resourceKey);
      }
@@ -534,7 +532,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
          resource.getClass().getSimpleName());
      return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
          .setKey(resourceKey)
-          .setLastUpdateTime(resource.getUpdateAutoTimestamp().getTimestamp())
+          .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
          .setRequestingClientId(
              checkNotNull(
                  params.get(PARAM_REQUESTING_CLIENT_ID), "Requesting client id not specified"))
@@ -207,9 +207,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
      Key<HostResource> referencingHostKey = null;
      for (DnsRefreshRequest request : refreshRequests) {
        if (isActive(domain, request.lastUpdateTime())
-            && domain
-                .getNameservers()
-                .contains(VKey.createOfy(HostResource.class, request.hostKey()))) {
+            && domain.getNameservers().contains(VKey.from(request.hostKey()))) {
          referencingHostKey = request.hostKey();
          break;
        }
@@ -321,13 +319,13 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
      HostResource host =
          checkNotNull(ofy().load().key(hostKey).now(), "Host to refresh doesn't exist");
      boolean isHostDeleted =
-          isDeleted(host, latestOf(now, host.getUpdateAutoTimestamp().getTimestamp()));
+          isDeleted(host, latestOf(now, host.getUpdateTimestamp().getTimestamp()));
      if (isHostDeleted) {
        logger.atInfo().log("Host %s is already deleted, not refreshing DNS.", hostKey);
      }
      return new AutoValue_RefreshDnsOnHostRenameAction_DnsRefreshRequest.Builder()
          .setHostKey(hostKey)
-          .setLastUpdateTime(host.getUpdateAutoTimestamp().getTimestamp())
+          .setLastUpdateTime(host.getUpdateTimestamp().getTimestamp())
          .setRequestedTime(
              DateTime.parse(
                  checkNotNull(params.get(PARAM_REQUESTED_TIME), "Requested time not specified")))
@@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Strings.isNullOrEmpty;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import org.joda.time.DateTime;

@@ -31,11 +32,28 @@ public final class BackupPaths {
  private BackupPaths() {}

  private static final String WILDCARD_CHAR = "*";
-  private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/input-%s";
+  private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/output-%s";

  public static final String COMMIT_LOG_NAME_PREFIX = "commit_diff_until_";
  private static final String COMMIT_LOG_PATTERN_TEMPLATE = "%s/" + COMMIT_LOG_NAME_PREFIX + "*";

+  /**
+   * Pattern of the per-project file with Cloud SQL connection information. To get a concrete path,
+   * user needs to provide the name of the environment, alpha, crash, sandbox, or production. This
+   * file is meant for applications without access to secrets stored in Datastore.
+   *
+   * <p>In production, this is an base-64 encoded encrypted file with one line, which contains
+   * space-separated values of Cloud SQL instance name, login, and password.
+   *
+   * <p>A plain text may be used for tests to a local database. Replace Cloud SQL instance name with
+   * JDBC URL.
+   */
+  private static final String SQL_CONN_INFO_FILE_PATTERN =
+      "gs://domain-registry-dev-deploy/cloudsql-credentials/%s/admin_credential.enc";
+
+  private static final ImmutableSet<String> ALLOWED_ENV =
+      ImmutableSet.of("alpha", "crash", "sandbox", "production");
+
  /**
   * Returns a regex pattern that matches all Datastore export files of a given {@code kind}.
   *
@@ -90,4 +108,10 @@ public final class BackupPaths {
    checkArgument(start >= 0, "Illegal file name %s.", fileName);
    return DateTime.parse(fileName.substring(start + COMMIT_LOG_NAME_PREFIX.length()));
  }
+
+  public static ImmutableList<String> getCloudSQLCredentialFilePatterns(String environmentName) {
+    checkArgument(
+        ALLOWED_ENV.contains(environmentName), "Invalid environment name %s", environmentName);
+    return ImmutableList.of(String.format(SQL_CONN_INFO_FILE_PATTERN, environmentName));
+  }
 }
@@ -0,0 +1,180 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import com.google.common.base.Splitter;
+import dagger.Component;
+import dagger.Lazy;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.keyring.kms.KmsModule;
+import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.JdbcJpaTm;
+import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.util.UtilsModule;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.channels.Channels;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import javax.annotation.Nullable;
+import javax.inject.Singleton;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResourceId;
+
+/**
+ * Provides bindings for {@link JpaTransactionManager} to Cloud SQL.
+ *
+ * <p>This module is intended for use in BEAM pipelines, and uses a BEAM utility to access GCS like
+ * a regular file system.
+ */
+@Module
+public class BeamJpaModule {
+
+  private static final String GCS_SCHEME = "gs://";
+
+  @Nullable private final String sqlAccessInfoFile;
+  @Nullable private final String cloudKmsProjectId;
+
+  /**
+   * Constructs a new instance of {@link BeamJpaModule}.
+   *
+   * <p>Note: it is an unfortunately necessary antipattern to check for the validity of
+   * sqlAccessInfoFile in {@link #provideCloudSqlAccessInfo} rather than in the constructor.
+   * Unfortunately, this is a restriction imposed upon us by Dagger. Specifically, because we use
+   * this in at least one 1 {@link google.registry.tools.RegistryTool} command(s), it must be
+   * instantiated in {@code google.registry.tools.RegistryToolComponent} for all possible commands;
+   * Dagger doesn't permit it to ever be null. For the vast majority of commands, it will never be
+   * used (so a null credential file path is fine in those cases).
+   *
+   * @param sqlAccessInfoFile the path to a Cloud SQL credential file. This must refer to either a
+   *     real encrypted file on GCS as returned by {@link
+   *     BackupPaths#getCloudSQLCredentialFilePatterns} or an unencrypted file on local filesystem
+   *     with credentials to a test database.
+   */
+  public BeamJpaModule(@Nullable String sqlAccessInfoFile, @Nullable String cloudKmsProjectId) {
+    this.sqlAccessInfoFile = sqlAccessInfoFile;
+    this.cloudKmsProjectId = cloudKmsProjectId;
+  }
+
+  /** Returns true if the credential file is on GCS (and therefore expected to be encrypted). */
+  private boolean isCloudSqlCredential() {
+    return sqlAccessInfoFile.startsWith(GCS_SCHEME);
+  }
+
+  @Provides
+  @Singleton
+  SqlAccessInfo provideCloudSqlAccessInfo(Lazy<CloudSqlCredentialDecryptor> lazyDecryptor) {
+    checkArgument(!isNullOrEmpty(sqlAccessInfoFile), "Null or empty credentialFilePath");
+    String line = readOnlyLineFromCredentialFile();
+    if (isCloudSqlCredential()) {
+      line = lazyDecryptor.get().decrypt(line);
+    }
+    // See ./BackupPaths.java for explanation of the line format.
+    List<String> parts = Splitter.on(' ').splitToList(line.trim());
+    checkState(parts.size() == 3, "Expecting three phrases in %s", line);
+    if (isCloudSqlCredential()) {
+      return SqlAccessInfo.createCloudSqlAccessInfo(parts.get(0), parts.get(1), parts.get(2));
+    } else {
+      return SqlAccessInfo.createLocalSqlAccessInfo(parts.get(0), parts.get(1), parts.get(2));
+    }
+  }
+
+  String readOnlyLineFromCredentialFile() {
+    try {
+      ResourceId resourceId = FileSystems.matchSingleFileSpec(sqlAccessInfoFile).resourceId();
+      try (BufferedReader reader =
+          new BufferedReader(
+              new InputStreamReader(
+                  Channels.newInputStream(FileSystems.open(resourceId)), StandardCharsets.UTF_8))) {
+        return reader.readLine();
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Provides
+  @Config("beamCloudSqlJdbcUrl")
+  String provideJdbcUrl(SqlAccessInfo sqlAccessInfo) {
+    return sqlAccessInfo.jdbcUrl();
+  }
+
+  @Provides
+  @Config("beamCloudSqlInstanceConnectionName")
+  String provideSqlInstanceName(SqlAccessInfo sqlAccessInfo) {
+    return sqlAccessInfo
+        .cloudSqlInstanceName()
+        .orElseThrow(() -> new IllegalStateException("Cloud SQL not provisioned."));
+  }
+
+  @Provides
+  @Config("beamCloudSqlUsername")
+  String provideSqlUsername(SqlAccessInfo sqlAccessInfo) {
+    return sqlAccessInfo.user();
+  }
+
+  @Provides
+  @Config("beamCloudSqlPassword")
+  String provideSqlPassword(SqlAccessInfo sqlAccessInfo) {
+    return sqlAccessInfo.password();
+  }
+
+  @Provides
+  @Config("beamCloudKmsProjectId")
+  String kmsProjectId() {
+    return cloudKmsProjectId;
+  }
+
+  @Provides
+  @Config("beamCloudKmsKeyRing")
+  static String keyRingName() {
+    return "nomulus-tool-keyring";
+  }
+
+  @Provides
+  @Config("beamHibernateHikariMaximumPoolSize")
+  static int getBeamHibernateHikariMaximumPoolSize() {
+    // TODO(weiminyu): make this configurable. Should be equal to number of cores.
+    return 4;
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        ConfigModule.class,
+        CredentialModule.class,
+        BeamJpaModule.class,
+        KmsModule.class,
+        PersistenceModule.class,
+        UtilsModule.class
+      })
+  public interface JpaTransactionManagerComponent {
+    @SocketFactoryJpaTm
+    JpaTransactionManager cloudSqlJpaTransactionManager();
+
+    @JdbcJpaTm
+    JpaTransactionManager localDbJpaTransactionManager();
+  }
+}
@@ -0,0 +1,50 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.api.services.cloudkms.v1.model.DecryptRequest;
+import com.google.common.base.Strings;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.keyring.kms.KmsConnection;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import javax.inject.Inject;
+
+/**
+ * Decrypts data using Cloud KMS, with the same crypto key with which Cloud SQL credential files on
+ * GCS was encrypted. See {@link BackupPaths#getCloudSQLCredentialFilePatterns} for more
+ * information.
+ */
+public class CloudSqlCredentialDecryptor {
+
+  private static final String CRYPTO_KEY_NAME = "nomulus-tool-key";
+  private final KmsConnection kmsConnection;
+
+  @Inject
+  CloudSqlCredentialDecryptor(@Config("beamKmsConnection") KmsConnection kmsConnection) {
+    this.kmsConnection = kmsConnection;
+  }
+
+  public String decrypt(String data) {
+    checkArgument(!Strings.isNullOrEmpty(data), "Null or empty data.");
+    byte[] ciphertext = Base64.getDecoder().decode(data);
+    // Re-encode for Cloud KMS JSON REST API, invoked through kmsConnection.
+    String urlSafeCipherText = new DecryptRequest().encodeCiphertext(ciphertext).getCiphertext();
+    return new String(
+        kmsConnection.decrypt(CRYPTO_KEY_NAME, urlSafeCipherText), StandardCharsets.UTF_8);
+  }
+}
@@ -0,0 +1,75 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.appengine.api.datastore.Entity;
+import java.util.Objects;
+
+/** Helper for manipulating {@code DomainBase} when migrating from Datastore to SQL database */
+final class DomainBaseUtil {
+
+  private DomainBaseUtil() {}
+
+  /**
+   * Removes {@link google.registry.model.billing.BillingEvent.Recurring}, {@link
+   * google.registry.model.poll.PollMessage PollMessages} and {@link
+   * google.registry.model.host.HostResource name servers} from a Datastore {@link Entity} that
+   * represents an Ofy {@link google.registry.model.domain.DomainBase}. This breaks the cycle of
+   * foreign key constraints between these entity kinds, allowing {@code DomainBases} to be inserted
+   * into the SQL database. See {@link InitSqlPipeline} for a use case, where the full {@code
+   * DomainBases} are written again during the last stage of the pipeline.
+   *
+   * <p>The returned object may be in bad state. Specifically, {@link
+   * google.registry.model.eppcommon.StatusValue#INACTIVE} is not added after name servers are
+   * removed. This only impacts tests.
+   *
+   * <p>This operation is performed on an Datastore {@link Entity} instead of Ofy Java object
+   * because Objectify requires access to a Datastore service when converting an Ofy object to a
+   * Datastore {@code Entity}. If we insist on working with Objectify objects, we face a few
+   * unsatisfactory options:
+   *
+   * <ul>
+   *   <li>Connect to our production Datastore, which incurs unnecessary security and code health
+   *       risk.
+   *   <li>Connect to a separate real Datastore instance, which is a waster and overkill.
+   *   <li>Use an in-memory test Datastore, which is a project health risk in that the test
+   *       Datastore would be added to Nomulus' production binary unless we create a separate
+   *       project for this pipeline.
+   * </ul>
+   *
+   * <p>Given our use case, operating on Datastore entities is the best option.
+   *
+   * @throws IllegalArgumentException if input does not represent a DomainBase
+   */
+  static Entity removeBillingAndPollAndHosts(Entity domainBase) {
+    checkNotNull(domainBase, "domainBase");
+    checkArgument(
+        Objects.equals(domainBase.getKind(), "DomainBase"),
+        "Expecting DomainBase, got %s",
+        domainBase.getKind());
+    Entity clone = domainBase.clone();
+    clone.removeProperty("autorenewBillingEvent");
+    clone.removeProperty("autorenewPollMessage");
+    clone.removeProperty("deletePollMessage");
+    clone.removeProperty("nsHosts");
+    domainBase.getProperties().keySet().stream()
+        .filter(s -> s.startsWith("transferData."))
+        .forEach(s -> clone.removeProperty(s));
+    return clone;
+  }
+}
@@ -0,0 +1,244 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.AppEngineEnvironment;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.BeamJpaModule.JpaTransactionManagerComponent;
+import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registry.Registry;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.Wait;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+
+/**
+ * A BEAM pipeline that populates a SQL database with data from a Datastore backup.
+ *
+ * <p>This pipeline migrates EPP resources and related entities that cross-reference each other. To
+ * avoid violating foreign key constraints, writes to SQL are ordered by entity kinds. In addition,
+ * the {@link DomainBase} kind is written twice (see details below). The write order is presented
+ * below. Although some kinds can be written concurrently, e.g. {@code ContactResource} and {@code
+ * RegistrarContact}, we do not expect any performance benefit since the limiting resource is the
+ * number of JDBC connections. Google internal users may refer to <a
+ * href="http://go/registry-r3-init-sql">the design doc</a> for more information.
+ *
+ * <ol>
+ *   <li>{@link Registry}: Assumes that {@code PremiumList} and {@code ReservedList} have been set
+ *       up in the SQL database.
+ *   <li>{@link Registrar}: Logically depends on {@code Registry}, Foreign key not modeled yet.
+ *   <li>{@link ContactResource}: references {@code Registrar}
+ *   <li>{@link RegistrarContact}: references {@code Registrar}.
+ *   <li>Cleansed {@link DomainBase}: with references to {@code BillingEvent}, {@code Recurring},
+ *       {@code Cancellation} and {@code HostResource} removed, still references {@code Registrar}
+ *       and {@code ContactResource}. The removal breaks circular Foreign Key references.
+ *   <li>{@link HostResource}: references {@code DomainBase}.
+ *   <li>{@link HistoryEntry}: maps to one of three SQL entity types and may reference {@code
+ *       Registrar}, {@code ContactResource}, {@code HostResource}, and {@code DomainBase}.
+ *   <li>{@link AllocationToken}: references {@code HistoryEntry}.
+ *   <li>{@link BillingEvent.Recurring}: references {@code Registrar}, {@code DomainBase} and {@code
+ *       HistoryEntry}.
+ *   <li>{@link BillingEvent.OneTime}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       BillingEvent.Recurring}, {@code HistoryEntry} and {@code AllocationToken}.
+ *   <li>{@link BillingEvent.Modification}: SQL model TBD. Will reference {@code Registrar}, {@code
+ *       DomainBase} and {@code BillingEvent.OneTime}.
+ *   <li>{@link BillingEvent.Cancellation}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       BillingEvent.Recurring}, {@code BillingEvent.OneTime}, and {@code HistoryEntry}.
+ *   <li>{@link PollMessage}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       ContactResource}, {@code HostResource}, and {@code HistoryEntry}.
+ *   <li>{@link DomainBase}, original copy from Datastore.
+ * </ol>
+ */
+public class InitSqlPipeline implements Serializable {
+
+  /**
+   * Datastore kinds to be written to the SQL database before the cleansed version of {@link
+   * DomainBase}.
+   */
+  // TODO(weiminyu): include Registry.class when it is modeled in JPA.
+  private static final ImmutableList<Class<?>> PHASE_ONE_ORDERED =
+      ImmutableList.of(Registrar.class, ContactResource.class);
+
+  /**
+   * Datastore kinds to be written to the SQL database after the cleansed version of {@link
+   * DomainBase}.
+   *
+   * <p>The following entities are missing from the list:
+   *
+   * <ul>
+   *   <li>Those not modeled in JPA yet, e.g., {@code BillingEvent.Modification}.
+   *   <li>Those waiting for sanitation, e.g., {@code HistoryEntry}, which would have duplicate keys
+   *       after converting to SQL model.
+   *   <li>Those that have foreign key constraints on the above.
+   * </ul>
+   */
+  // TODO(weiminyu): add more entities when available.
+  private static final ImmutableList<Class<?>> PHASE_TWO_ORDERED =
+      ImmutableList.of(HostResource.class);
+
+  private final InitSqlPipelineOptions options;
+
+  private final Pipeline pipeline;
+
+  private final SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager>
+      jpaGetter;
+
+  InitSqlPipeline(InitSqlPipelineOptions options) {
+    this.options = options;
+    pipeline = Pipeline.create(options);
+    jpaGetter = JpaTransactionManagerComponent::cloudSqlJpaTransactionManager;
+  }
+
+  @VisibleForTesting
+  InitSqlPipeline(InitSqlPipelineOptions options, Pipeline pipeline) {
+    this.options = options;
+    this.pipeline = pipeline;
+    jpaGetter = JpaTransactionManagerComponent::localDbJpaTransactionManager;
+  }
+
+  public PipelineResult run() {
+    setupPipeline();
+    return pipeline.run();
+  }
+
+  @VisibleForTesting
+  void setupPipeline() {
+    PCollectionTuple datastoreSnapshot =
+        pipeline.apply(
+            "Load Datastore snapshot",
+            Transforms.loadDatastoreSnapshot(
+                options.getDatastoreExportDir(),
+                options.getCommitLogDir(),
+                DateTime.parse(options.getCommitLogStartTimestamp()),
+                DateTime.parse(options.getCommitLogEndTimestamp()),
+                ImmutableSet.<String>builder()
+                    .add("DomainBase")
+                    .addAll(toKindStrings(PHASE_ONE_ORDERED))
+                    .addAll(toKindStrings(PHASE_TWO_ORDERED))
+                    .build()));
+
+    // Set up the pipeline to write entity kinds from PHASE_ONE_ORDERED to SQL. Return a object
+    // that signals the completion of the phase.
+    PCollection<Void> blocker =
+        scheduleOnePhaseWrites(datastoreSnapshot, PHASE_ONE_ORDERED, Optional.empty(), null);
+    blocker =
+        writeToSql(
+            "DomainBase without circular foreign keys",
+            removeDomainBaseForeignKeys(datastoreSnapshot)
+                .apply("Wait on phase one", Wait.on(blocker)));
+    // Set up the pipeline to write entity kinds from PHASE_TWO_ORDERED to SQL. This phase won't
+    // start until all cleansed DomainBases have been written (started by line above).
+    scheduleOnePhaseWrites(
+        datastoreSnapshot, PHASE_TWO_ORDERED, Optional.of(blocker), "DomainBaseNoFkeys");
+  }
+
+  private PCollection<VersionedEntity> removeDomainBaseForeignKeys(
+      PCollectionTuple datastoreSnapshot) {
+    PCollection<VersionedEntity> domainBases =
+        datastoreSnapshot.get(Transforms.createTagForKind("DomainBase"));
+    return domainBases.apply(
+        "Remove circular foreign keys from DomainBase",
+        ParDo.of(new RemoveDomainBaseForeignKeys()));
+  }
+
+  /**
+   * Sets up the pipeline to write entities in {@code entityClasses} to SQL. Entities are written
+   * one kind at a time based on each kind's position in {@code entityClasses}. Concurrency exists
+   * within each kind.
+   *
+   * @param datastoreSnapshot the Datastore snapshot of all data to be migrated to SQL
+   * @param entityClasses the entity types in write order
+   * @param blockingPCollection the pipeline stage that blocks this phase
+   * @param blockingTag description of the stage (if exists) that blocks this phase. Needed for
+   *     generating unique transform ids
+   * @return the output {@code PCollection} from the writing of the last entity kind. Other parts of
+   *     the pipeline can {@link Wait} on this object
+   */
+  private PCollection<Void> scheduleOnePhaseWrites(
+      PCollectionTuple datastoreSnapshot,
+      Collection<Class<?>> entityClasses,
+      Optional<PCollection<Void>> blockingPCollection,
+      String blockingTag) {
+    checkArgument(!entityClasses.isEmpty(), "Each phase must have at least one kind.");
+    ImmutableList<TupleTag<VersionedEntity>> tags =
+        toKindStrings(entityClasses).stream()
+            .map(Transforms::createTagForKind)
+            .collect(ImmutableList.toImmutableList());
+
+    PCollection<Void> prev = blockingPCollection.orElse(null);
+    String prevTag = blockingTag;
+    for (TupleTag<VersionedEntity> tag : tags) {
+      PCollection<VersionedEntity> curr = datastoreSnapshot.get(tag);
+      if (prev != null) {
+        curr = curr.apply("Wait on " + prevTag, Wait.on(prev));
+      }
+      prev = writeToSql(tag.getId(), curr);
+      prevTag = tag.getId();
+    }
+    return prev;
+  }
+
+  private PCollection<Void> writeToSql(String transformId, PCollection<VersionedEntity> data) {
+    String credentialFileUrl =
+        options.getSqlCredentialUrlOverride() != null
+            ? options.getSqlCredentialUrlOverride()
+            : BackupPaths.getCloudSQLCredentialFilePatterns(options.getEnvironment()).get(0);
+
+    return data.apply(
+        "Write to sql: " + transformId,
+        Transforms.writeToSql(
+            transformId,
+            options.getMaxConcurrentSqlWriters(),
+            options.getSqlWriteBatchSize(),
+            new JpaSupplierFactory(credentialFileUrl, options.getCloudKmsProjectId(), jpaGetter)));
+  }
+
+  private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
+    try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+      return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
+    }
+  }
+
+  public static void main(String[] args) {
+    InitSqlPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(InitSqlPipelineOptions.class);
+    new InitSqlPipeline(options).run();
+  }
+}
@@ -0,0 +1,91 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Validation;
+
+/** Pipeline options for {@link InitSqlPipeline} */
+public interface InitSqlPipelineOptions extends GcpOptions {
+
+  @Description(
+      "Overrides the URL to the SQL credential file. " + "Required if environment is not provided.")
+  @Nullable
+  String getSqlCredentialUrlOverride();
+
+  void setSqlCredentialUrlOverride(String credentialUrlOverride);
+
+  @Description("The root directory of the export to load.")
+  String getDatastoreExportDir();
+
+  void setDatastoreExportDir(String datastoreExportDir);
+
+  @Description("The directory that contains all CommitLog files.")
+  String getCommitLogDir();
+
+  void setCommitLogDir(String commitLogDir);
+
+  @Description("The earliest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getCommitLogStartTimestamp();
+
+  void setCommitLogStartTimestamp(String commitLogStartTimestamp);
+
+  @Description("The latest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getCommitLogEndTimestamp();
+
+  void setCommitLogEndTimestamp(String commitLogEndTimestamp);
+
+  @Description(
+      "The deployed environment, alpha, crash, sandbox, or production. "
+          + "Not required only if sqlCredentialUrlOverride is provided.")
+  @Nullable
+  String getEnvironment();
+
+  void setEnvironment(String environment);
+
+  @Description(
+      "The GCP project that contains the keyring used for decrypting the " + "SQL credential file.")
+  @Nullable
+  String getCloudKmsProjectId();
+
+  void setCloudKmsProjectId(String cloudKmsProjectId);
+
+  @Description(
+      "The maximum JDBC connection pool size on a VM. "
+          + "This value should be equal to or greater than the number of cores on the VM.")
+  @Default.Integer(4)
+  int getJdbcMaxPoolSize();
+
+  void setJdbcMaxPoolSize(int jdbcMaxPoolSize);
+
+  @Description(
+      "A hint to the pipeline runner of the maximum number of concurrent SQL writers to create. "
+          + "Note that multiple writers may run on the same VM and share the connection pool.")
+  @Default.Integer(4)
+  int getMaxConcurrentSqlWriters();
+
+  void setMaxConcurrentSqlWriters(int maxConcurrentSqlWriters);
+
+  @Description("The number of entities to be written to the SQL database in one transaction.")
+  @Default.Integer(20)
+  int getSqlWriteBatchSize();
+
+  void setSqlWriteBatchSize(int sqlWriteBatchSize);
+}
@@ -0,0 +1,48 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import google.registry.beam.initsql.BeamJpaModule.JpaTransactionManagerComponent;
+import google.registry.beam.initsql.Transforms.SerializableSupplier;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+
+public class JpaSupplierFactory implements SerializableSupplier<JpaTransactionManager> {
+
+  private static final long serialVersionUID = 1L;
+
+  private final String credentialFileUrl;
+  @Nullable private final String cloudKmsProjectId;
+  private final SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager>
+      jpaGetter;
+
+  public JpaSupplierFactory(
+      String credentialFileUrl,
+      @Nullable String cloudKmsProjectId,
+      SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager> jpaGetter) {
+    this.credentialFileUrl = credentialFileUrl;
+    this.cloudKmsProjectId = cloudKmsProjectId;
+    this.jpaGetter = jpaGetter;
+  }
+
+  @Override
+  public JpaTransactionManager get() {
+    return jpaGetter.apply(
+        DaggerBeamJpaModule_JpaTransactionManagerComponent.builder()
+            .beamJpaModule(new BeamJpaModule(credentialFileUrl, cloudKmsProjectId))
+            .build());
+  }
+}
@@ -1,3 +1,17 @@
 ## Summary

-This package contains a BEAM pipeline that populates a Cloud SQL database from a Datastore backup.
+This package contains a BEAM pipeline that populates a Cloud SQL database from a
+Datastore backup. The pipeline uses an unsynchronized Datastore export and
+overlapping CommitLogs generated by the Nomulus server to recreate a consistent
+Datastore snapshot, and writes the data to a Cloud SQL instance.
+
+## Pipeline Visualization
+
+The golden flow graph of the InitSqlPipeline is saved both as a text-base
+[DOT file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.dot)
+and a
+[.png file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.png).
+A test compares the flow graph of the current pipeline with the golden graph,
+and will fail if changes are detected. When this happens, run the Gradle task
+':core:updateInitSqlPipelineGraph' to update the golden files and review the
+changes.
@@ -0,0 +1,45 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import com.google.auto.value.AutoValue;
+import java.util.Optional;
+
+/**
+ * Information needed to connect to a database, including JDBC URL, user name, password, and in the
+ * case of Cloud SQL, the database instance's name.
+ */
+@AutoValue
+abstract class SqlAccessInfo {
+
+  abstract String jdbcUrl();
+
+  abstract String user();
+
+  abstract String password();
+
+  abstract Optional<String> cloudSqlInstanceName();
+
+  public static SqlAccessInfo createCloudSqlAccessInfo(
+      String sqlInstanceName, String username, String password) {
+    return new AutoValue_SqlAccessInfo(
+        "jdbc:postgresql://google/postgres", username, password, Optional.of(sqlInstanceName));
+  }
+
+  public static SqlAccessInfo createLocalSqlAccessInfo(
+      String jdbcUrl, String username, String password) {
+    return new AutoValue_SqlAccessInfo(jdbcUrl, username, password, Optional.empty());
+  }
+}
@@ -16,18 +16,43 @@ package google.registry.beam.initsql;

 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Throwables.throwIfUnchecked;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
+import static google.registry.persistence.JpaRetries.isFailedTxnRetriable;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.setJpaTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static java.util.Comparator.comparing;
+import static org.apache.beam.sdk.values.TypeDescriptors.integers;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;

 import avro.shaded.com.google.common.collect.Iterators;
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
+import google.registry.backup.AppEngineEnvironment;
 import google.registry.backup.CommitLogImports;
 import google.registry.backup.VersionedEntity;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.ofy.Ofy;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.tools.LevelDbLogReader;
+import google.registry.util.SystemSleeper;
+import java.io.Serializable;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Supplier;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.Compression;
 import org.apache.beam.sdk.io.FileIO;
@@ -36,12 +61,23 @@ import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.GroupIntoBatches;
+import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ProcessFunction;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.TypeDescriptor;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;

 /**
 * {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export
@@ -57,6 +93,110 @@ public final class Transforms {
   */
  @VisibleForTesting static final long EXPORT_ENTITY_TIME_STAMP = START_OF_TIME.getMillis();

+  /**
+   * Returns a {@link TupleTag} that can be used to retrieve entities of the given {@code kind} from
+   * the Datastore snapshot returned by {@link #loadDatastoreSnapshot}.
+   */
+  public static TupleTag<VersionedEntity> createTagForKind(String kind) {
+    // When used with PCollectionTuple the result must retain generic type information.
+    // Both the Generic param and the empty bracket below are important.
+    return new TupleTag<VersionedEntity>(Transforms.class.getSimpleName() + ":" + kind) {};
+  }
+
+  /**
+   * Composite {@link PTransform transform} that loads the Datastore snapshot at {@code
+   * commitLogToTime} for caller specified {@code kinds}.
+   *
+   * <p>Caller must provide the location of a Datastore export that started AFTER {@code
+   * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory
+   * of all CommitLog files.
+   *
+   * <p>Selection of {@code commitLogFromTime} and {@code commitLogToTime} should follow the
+   * guidelines below to ensure that all incremental changes concurrent with the export are covered:
+   *
+   * <ul>
+   *   <li>Two or more CommitLogs should exist between {@code commitLogFromTime} and the starting
+   *       time of the Datastore export. This ensures that the earlier CommitLog file was complete
+   *       before the export started.
+   *   <li>Two or more CommitLogs should exit between the export completion time and {@code
+   *       commitLogToTime}.
+   * </ul>
+   *
+   * <p>The output from the returned transform is a {@link PCollectionTuple} consisting of {@link
+   * VersionedEntity VersionedEntities} grouped into {@link PCollection PCollections} by {@code
+   * kind}.
+   */
+  public static PTransform<PBegin, PCollectionTuple> loadDatastoreSnapshot(
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<String> kinds) {
+    checkArgument(kinds != null && !kinds.isEmpty(), "At least one kind is expected.");
+
+    // Create tags to collect entities by kind in final step.
+    final ImmutableMap<String, TupleTag<VersionedEntity>> outputTags =
+        kinds.stream()
+            .collect(ImmutableMap.toImmutableMap(kind -> kind, Transforms::createTagForKind));
+    // Arbitrarily select one tag as mainOutTag and put the remaining ones in a TupleTagList.
+    // This separation is required by ParDo's config API.
+    Iterator<TupleTag<VersionedEntity>> tagsIt = outputTags.values().iterator();
+    final TupleTag<VersionedEntity> mainOutputTag = tagsIt.next();
+    final TupleTagList additionalTags = TupleTagList.of(ImmutableList.copyOf(tagsIt));
+
+    return new PTransform<PBegin, PCollectionTuple>() {
+      @Override
+      public PCollectionTuple expand(PBegin input) {
+        PCollection<VersionedEntity> exportedEntities =
+            input
+                .apply("Get export file patterns", getDatastoreExportFilePatterns(exportDir, kinds))
+                .apply("Find export files", getFilesByPatterns())
+                .apply("Load export data", loadExportDataFromFiles());
+        PCollection<VersionedEntity> commitLogEntities =
+            input
+                .apply("Get commitlog file patterns", getCommitLogFilePatterns(commitLogDir))
+                .apply("Find commitlog files", getFilesByPatterns())
+                .apply(
+                    "Filter commitLog by time",
+                    filterCommitLogsByTime(commitLogFromTime, commitLogToTime))
+                .apply("Load commitlog data", loadCommitLogsFromFiles(kinds));
+        return PCollectionList.of(exportedEntities)
+            .and(commitLogEntities)
+            .apply("Merge exports and CommitLogs", Flatten.pCollections())
+            .apply(
+                "Key entities by Datastore Keys",
+                // Converting to KV<String, VE> instead of KV<Key, VE> b/c default coder for Key
+                // (SerializableCoder) is not deterministic and cannot be used with GroupBy.
+                MapElements.into(kvs(strings(), TypeDescriptor.of(VersionedEntity.class)))
+                    .via((VersionedEntity e) -> KV.of(e.key().toString(), e)))
+            .apply("Gather entities by key", GroupByKey.create())
+            .apply(
+                "Output latest version per entity",
+                ParDo.of(
+                        new DoFn<KV<String, Iterable<VersionedEntity>>, VersionedEntity>() {
+                          @ProcessElement
+                          public void processElement(
+                              @Element KV<String, Iterable<VersionedEntity>> kv,
+                              MultiOutputReceiver out) {
+                            Optional<VersionedEntity> latest =
+                                Streams.stream(kv.getValue())
+                                    .sorted(comparing(VersionedEntity::commitTimeMills).reversed())
+                                    .findFirst();
+                            // Throw to abort (after default retries). Investigate, fix, and rerun.
+                            checkState(
+                                latest.isPresent(), "Unexpected key with no data", kv.getKey());
+                            if (latest.get().isDelete()) {
+                              return;
+                            }
+                            String kind = latest.get().getEntity().get().getKind();
+                            out.get(outputTags.get(kind)).output(latest.get());
+                          }
+                        })
+                    .withOutputTags(mainOutputTag, additionalTags));
+      }
+    };
+  }
+
  /**
   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
   * all Datastore CommitLog files.
@@ -75,6 +215,11 @@ public final class Transforms {
    return toStringPCollection(getExportFilePatterns(exportDir, kinds));
  }

+  public static PTransform<PBegin, PCollection<String>> getCloudSqlConnectionInfoFilePatterns(
+      String gcpProjectName) {
+    return toStringPCollection(BackupPaths.getCloudSQLCredentialFilePatterns(gcpProjectName));
+  }
+
  /**
   * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}.
   */
@@ -82,7 +227,7 @@ public final class Transforms {
    return new PTransform<PCollection<String>, PCollection<Metadata>>() {
      @Override
      public PCollection<Metadata> expand(PCollection<String> input) {
-        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW));
      }
    };
  }
@@ -91,7 +236,7 @@ public final class Transforms {
   * Returns CommitLog files with timestamps between {@code fromTime} (inclusive) and {@code
   * endTime} (exclusive).
   */
-  public static PTransform<PCollection<? extends String>, PCollection<String>>
+  public static PTransform<PCollection<? extends Metadata>, PCollection<Metadata>>
      filterCommitLogsByTime(DateTime fromTime, DateTime toTime) {
    return ParDo.of(new FilterCommitLogFileByTime(fromTime, toTime));
  }
@@ -109,11 +254,51 @@ public final class Transforms {

  /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
-      loadCommitLogsFromFiles() {
+      loadCommitLogsFromFiles(Set<String> kinds) {
    return processFiles(
-        new BackupFileReader(file -> CommitLogImports.loadEntities(file.open()).iterator()));
+        new BackupFileReader(
+            file ->
+                CommitLogImports.loadEntities(file.open()).stream()
+                    .filter(e -> kinds.contains(e.key().getKind()))
+                    .iterator()));
  }

+  /**
+   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
+   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
+   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   *
+   * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
+   * job, we will not add features unless proven necessary.
+   *
+   * @param transformId a unique ID for an instance of the returned transform
+   * @param maxWriters the max number of concurrent writes to SQL, which also determines the max
+   *     number of connection pools created
+   * @param batchSize the number of entities to write in each operation
+   * @param jpaSupplier supplier of a {@link JpaTransactionManager}
+   */
+  public static PTransform<PCollection<VersionedEntity>, PCollection<Void>> writeToSql(
+      String transformId,
+      int maxWriters,
+      int batchSize,
+      SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+    return new PTransform<PCollection<VersionedEntity>, PCollection<Void>>() {
+      @Override
+      public PCollection<Void> expand(PCollection<VersionedEntity> input) {
+        return input
+            .apply(
+                "Shard data for " + transformId,
+                MapElements.into(kvs(integers(), TypeDescriptor.of(VersionedEntity.class)))
+                    .via(ve -> KV.of(ThreadLocalRandom.current().nextInt(maxWriters), ve)))
+            .apply("Batch output by shard " + transformId, GroupIntoBatches.ofSize(batchSize))
+            .apply("Write in batch for " + transformId, ParDo.of(new SqlBatchWriter(jpaSupplier)));
+      }
+    };
+  }
+
+  /** Interface for serializable {@link Supplier suppliers}. */
+  public interface SerializableSupplier<T> extends Supplier<T>, Serializable {}
+
  /**
   * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
   * given {@link Iterable}.
@@ -134,12 +319,11 @@ public final class Transforms {
        return input
            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
            .apply(transformer.getClass().getSimpleName(), ParDo.of(transformer));
-        // TODO(weiminyu): reshuffle to enable dynamic work rebalance per beam dev guide
      }
    };
  }

-  private static class FilterCommitLogFileByTime extends DoFn<String, String> {
+  private static class FilterCommitLogFileByTime extends DoFn<Metadata, Metadata> {
    private final DateTime fromTime;
    private final DateTime toTime;

@@ -156,10 +340,10 @@ public final class Transforms {
    }

    @ProcessElement
-    public void processElement(@Element String fileName, OutputReceiver<String> out) {
-      DateTime timestamp = getCommitLogTimestamp(fileName);
+    public void processElement(@Element Metadata fileMeta, OutputReceiver<Metadata> out) {
+      DateTime timestamp = getCommitLogTimestamp(fileMeta.resourceId().toString());
      if (isBeforeOrAt(fromTime, timestamp) && timestamp.isBefore(toTime)) {
-        out.output(fileName);
+        out.output(fileMeta);
      }
    }
  }
@@ -192,4 +376,116 @@ public final class Transforms {
      }
    }
  }
+
+  /**
+   * Writes a batch of entities to a SQL database.
+   *
+   * <p>Note that an arbitrary number of instances of this class may be created and freed in
+   * arbitrary order in a single JVM. Due to the tech debt that forced us to use a static variable
+   * to hold the {@code JpaTransactionManager} instance, we must ensure that JpaTransactionManager
+   * is not changed or torn down while being used by some instance.
+   */
+  private static class SqlBatchWriter extends DoFn<KV<Integer, Iterable<VersionedEntity>>, Void> {
+
+    private static int instanceCount = 0;
+    private static JpaTransactionManager originalJpa;
+
+    private final SerializableSupplier<JpaTransactionManager> jpaSupplier;
+
+    private transient Ofy ofy;
+    private transient SystemSleeper sleeper;
+
+    SqlBatchWriter(SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+      this.jpaSupplier = jpaSupplier;
+    }
+
+    @Setup
+    public void setup() {
+      sleeper = new SystemSleeper();
+
+      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+        ObjectifyService.initOfy();
+        ofy = ObjectifyService.ofy();
+      }
+
+      synchronized (SqlBatchWriter.class) {
+        if (instanceCount == 0) {
+          originalJpa = jpaTm();
+          setJpaTm(jpaSupplier);
+        }
+        instanceCount++;
+      }
+    }
+
+    @Teardown
+    public void teardown() {
+      synchronized (SqlBatchWriter.class) {
+        instanceCount--;
+        if (instanceCount == 0) {
+          jpaTm().teardown();
+          setJpaTm(() -> originalJpa);
+        }
+      }
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<Integer, Iterable<VersionedEntity>> kv) {
+      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+        ImmutableList<Object> ofyEntities =
+            Streams.stream(kv.getValue())
+                .map(VersionedEntity::getEntity)
+                .map(Optional::get)
+                .map(ofy::toPojo)
+                .collect(ImmutableList.toImmutableList());
+        retry(() -> jpaTm().transact(() -> jpaTm().saveNewOrUpdateAll(ofyEntities)));
+      }
+    }
+
+    // TODO(b/160632289): Enhance Retrier and use it here.
+    private void retry(Runnable runnable) {
+      int maxAttempts = 5;
+      int initialDelayMillis = 100;
+      double jitterRatio = 0.2;
+
+      for (int attempt = 0; attempt < maxAttempts; attempt++) {
+        try {
+          runnable.run();
+          return;
+        } catch (Throwable throwable) {
+          if (!isFailedTxnRetriable(throwable)) {
+            throwIfUnchecked(throwable);
+            throw new RuntimeException(throwable);
+          }
+          int sleepMillis = (1 << attempt) * initialDelayMillis;
+          int jitter =
+              ThreadLocalRandom.current().nextInt((int) (sleepMillis * jitterRatio))
+                  - (int) (sleepMillis * jitterRatio / 2);
+          sleeper.sleepUninterruptibly(Duration.millis(sleepMillis + jitter));
+        }
+      }
+    }
+  }
+
+  /**
+   * Removes BillingEvents, {@link google.registry.model.poll.PollMessage PollMessages} and {@link
+   * google.registry.model.host.HostResource} from a {@link DomainBase}. These are circular foreign
+   * key constraints that prevent migration of {@code DomainBase} to SQL databases.
+   *
+   * <p>See {@link InitSqlPipeline} for more information.
+   */
+  static class RemoveDomainBaseForeignKeys extends DoFn<VersionedEntity, VersionedEntity> {
+
+    @ProcessElement
+    public void processElement(
+        @Element VersionedEntity domainBase, OutputReceiver<VersionedEntity> out) {
+      checkArgument(
+          domainBase.getEntity().isPresent(), "Unexpected delete entity %s", domainBase.key());
+      Entity outputEntity =
+          DomainBaseUtil.removeBillingAndPollAndHosts(domainBase.getEntity().get());
+      out.output(
+          VersionedEntity.from(
+              domainBase.commitTimeMills(),
+              EntityTranslator.convertToPb(outputEntity).toByteArray()));
+    }
+  }
 }
@@ -235,6 +235,7 @@ public abstract class BillingEvent implements Serializable {
                DATE_TIME_FORMATTER.format(eventTime()),
                registrarId(),
                billingId(),
+                poNumber(),
                tld(),
                action(),
                domain(),
@@ -56,7 +56,8 @@ FROM (
      `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
    WHERE
    -- TODO(b/18092292): Add a filter for tldState (not PDT/PREDELEGATION)
-      tldType = 'REAL') ) AS BillingEvent
+      tldType = 'REAL'
+    AND disableInvoicing is not TRUE) ) AS BillingEvent
  -- Gather billing ID from registrar table
  -- This is a 'JOIN' as opposed to 'LEFT JOIN' to filter out
  -- non-billable registrars
@@ -141,7 +141,7 @@ public class SafeBrowsingTransforms {
    @ProcessElement
    public void processElement(ProcessContext context) {
      Subdomain subdomain = context.element();
-      subdomainBuffer.put(subdomain.fullyQualifiedDomainName(), subdomain);
+      subdomainBuffer.put(subdomain.domainName(), subdomain);
      if (subdomainBuffer.size() >= BATCH_SIZE) {
        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
        results.forEach(context::output);
@@ -239,7 +239,7 @@ public class SafeBrowsingTransforms {
            String url = match.getJSONObject("threat").getString("url");
            Subdomain subdomain = subdomainBuffer.get(url);
            resultBuilder.add(
-                KV.of(subdomain, ThreatMatch.create(match, subdomain.fullyQualifiedDomainName())));
+                KV.of(subdomain, ThreatMatch.create(match, subdomain.domainName())));
          }
        }
      }
@@ -19,9 +19,15 @@ import static google.registry.beam.BeamUtils.getQueryFromFile;

 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableSet;
+import google.registry.backup.AppEngineEnvironment;
+import google.registry.beam.initsql.Transforms.SerializableSupplier;
 import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
 import google.registry.config.CredentialModule.LocalCredential;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.model.reporting.Spec11ThreatMatch;
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
 import google.registry.util.SqlTemplate;
@@ -37,6 +43,7 @@ import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -46,6 +53,7 @@ import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.sdk.values.TypeDescriptors;
 import org.joda.time.LocalDate;
 import org.joda.time.YearMonth;
+import org.joda.time.format.ISODateTimeFormat;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -77,7 +85,7 @@ public class Spec11Pipeline implements Serializable {
  public static final String REGISTRAR_EMAIL_FIELD = "registrarEmailAddress";
  /** The JSON object field into which we put the registrar's name for Spec11 reports. */
  public static final String REGISTRAR_CLIENT_ID_FIELD = "registrarClientId";
-  /** The JSON object field we put the threat match array for Spec11 reports. */
+  /** The JSON object field into which we put the threat match array for Spec11 reports. */
  public static final String THREAT_MATCHES_FIELD = "threatMatches";

  private final String projectId;
@@ -86,6 +94,7 @@ public class Spec11Pipeline implements Serializable {
  private final String reportingBucketUrl;
  private final GoogleCredentials googleCredentials;
  private final Retrier retrier;
+  private final SerializableSupplier<JpaTransactionManager> jpaSupplierFactory;

  @Inject
  public Spec11Pipeline(
@@ -93,13 +102,14 @@ public class Spec11Pipeline implements Serializable {
      @Config("beamStagingUrl") String beamStagingUrl,
      @Config("spec11TemplateUrl") String spec11TemplateUrl,
      @Config("reportingBucketUrl") String reportingBucketUrl,
+      SerializableSupplier<JpaTransactionManager> jpaSupplierFactory,
      @LocalCredential GoogleCredentialsBundle googleCredentialsBundle,
-      Retrier retrier
-  ) {
+      Retrier retrier) {
    this.projectId = projectId;
    this.beamStagingUrl = beamStagingUrl;
    this.spec11TemplateUrl = spec11TemplateUrl;
    this.reportingBucketUrl = reportingBucketUrl;
+    this.jpaSupplierFactory = jpaSupplierFactory;
    this.googleCredentials = googleCredentialsBundle.getGoogleCredentials();
    this.retrier = retrier;
  }
@@ -177,18 +187,48 @@ public class Spec11Pipeline implements Serializable {
      PCollection<Subdomain> domains,
      EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
      ValueProvider<String> dateProvider) {
-    PCollection<KV<Subdomain, ThreatMatch>> subdomains =
+
+    PCollection<KV<Subdomain, ThreatMatch>> subdomainsSql =
+        domains.apply("Run through SafeBrowsing API", ParDo.of(evaluateSafeBrowsingFn));
+    /* Store ThreatMatch objects in SQL. */
+    subdomainsSql.apply(
+        ParDo.of(
+            new DoFn<KV<Subdomain, ThreatMatch>, Void>() {
+              @ProcessElement
+              public void processElement(ProcessContext context) {
+                // create the Spec11ThreatMatch from Subdomain and ThreatMatch
+                try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+                  Subdomain subdomain = context.element().getKey();
+                  Spec11ThreatMatch threatMatch =
+                      new Spec11ThreatMatch.Builder()
+                          .setThreatTypes(
+                              ImmutableSet.of(
+                                  ThreatType.valueOf(context.element().getValue().threatType())))
+                          .setCheckDate(
+                              LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
+                          .setDomainName(subdomain.domainName())
+                          .setDomainRepoId(subdomain.domainRepoId())
+                          .setRegistrarId(subdomain.registrarId())
+                          .build();
+                  JpaTransactionManager jpaTransactionManager = jpaSupplierFactory.get();
+                  jpaTransactionManager.transact(() -> jpaTransactionManager.saveNew(threatMatch));
+                }
+              }
+            }));
+
+    /* Store ThreatMatch objects in JSON. */
+    PCollection<KV<Subdomain, ThreatMatch>> subdomainsJson =
        domains.apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn));
-    subdomains
+    subdomainsJson
        .apply(
-            "Map registrar client ID to email/ThreatMatch pair",
+            "Map registrar ID to email/ThreatMatch pair",
            MapElements.into(
                    TypeDescriptors.kvs(
                        TypeDescriptors.strings(), TypeDescriptor.of(EmailAndThreatMatch.class)))
                .via(
                    (KV<Subdomain, ThreatMatch> kv) ->
                        KV.of(
-                            kv.getKey().registrarClientId(),
+                            kv.getKey().registrarId(),
                            EmailAndThreatMatch.create(
                                kv.getKey().registrarEmailAddress(), kv.getValue()))))
        .apply("Group by registrar client ID", GroupByKey.create())
@@ -36,12 +36,14 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 public abstract class Subdomain implements Serializable {

  private static final ImmutableList<String> FIELD_NAMES =
-      ImmutableList.of("fullyQualifiedDomainName", "registrarClientId", "registrarEmailAddress");
+      ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress");

  /** Returns the fully qualified domain name. */
-  abstract String fullyQualifiedDomainName();
-  /** Returns the client ID of the associated registrar for this domain. */
-  abstract String registrarClientId();
+  abstract String domainName();
+  /** Returns the domain repo ID (the primary key of the domain table). */
+  abstract String domainRepoId();
+  /** Returns the registrar ID of the associated registrar for this domain. */
+  abstract String registrarId();
  /** Returns the email address of the registrar associated with this domain. */
  abstract String registrarEmailAddress();

@@ -56,8 +58,9 @@ public abstract class Subdomain implements Serializable {
    checkFieldsNotNull(FIELD_NAMES, schemaAndRecord);
    GenericRecord record = schemaAndRecord.getRecord();
    return create(
-        extractField(record, "fullyQualifiedDomainName"),
-        extractField(record, "registrarClientId"),
+        extractField(record, "domainName"),
+        extractField(record, "domainRepoId"),
+        extractField(record, "registrarId"),
        extractField(record, "registrarEmailAddress"));
  }

@@ -69,9 +72,11 @@ public abstract class Subdomain implements Serializable {
   */
  @VisibleForTesting
  static Subdomain create(
-      String fullyQualifiedDomainName, String registrarClientId, String registrarEmailAddress) {
+      String domainName,
+      String domainRepoId,
+      String registrarId,
+      String registrarEmailAddress) {
    return new AutoValue_Subdomain(
-        fullyQualifiedDomainName, registrarClientId, registrarEmailAddress);
+        domainName, domainRepoId, registrarId, registrarEmailAddress);
  }
 }
-
@@ -19,11 +19,13 @@
  -- email address.

 SELECT
-  domain.fullyQualifiedDomainName AS fullyQualifiedDomainName,
-  registrar.clientId AS registrarClientId,
+  domain.fullyQualifiedDomainName AS domainName,
+  domain.__key__.name AS domainRepoId,
+  registrar.clientId AS registrarId,
  COALESCE(registrar.emailAddress, '') AS registrarEmailAddress
 FROM ( (
    SELECT
+      __key__,
      fullyQualifiedDomainName,
      currentSponsorClientId,
      creationTime
@@ -1527,6 +1527,21 @@ public final class RegistryConfig {
    return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
  }

+  /**
+   * Returns whether to replicate cloud SQL transactions to datastore.
+   *
+   * <p>If true, all cloud SQL transactions will be persisted as TransactionEntity objects in the
+   * Transaction table and replayed against datastore in a cron job.
+   */
+  public static boolean getCloudSqlReplicateTransactions() {
+    return CONFIG_SETTINGS.get().cloudSql.replicateTransactions;
+  }
+
+  @VisibleForTesting
+  public static void overrideCloudSqlReplicateTransactions(boolean replicateTransactions) {
+    CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
+  }
+
  /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
  public static String getContactAndHostRoidSuffix() {
    return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;
@@ -122,6 +122,7 @@ public class RegistryConfigSettings {
    public String jdbcUrl;
    public String username;
    public String instanceConnectionName;
+    public boolean replicateTransactions;
  }

  /** Configuration for Apache Beam (Cloud Dataflow). */
@@ -230,6 +230,9 @@ cloudSql:
  username: username
  # This name is used by Cloud SQL when connecting to the database.
  instanceConnectionName: project-id:region:instance-id
+  # Set this to true to replicate cloud SQL transactions to datastore in the
+  # background.
+  replicateTransactions: false

 cloudDns:
  # Set both properties to null in Production.
@@ -1,275 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.documentation;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.MoreCollectors.onlyElement;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Multimaps;
-import com.sun.javadoc.AnnotationDesc;
-import com.sun.javadoc.ClassDoc;
-import com.sun.javadoc.FieldDoc;
-import com.sun.javadoc.SeeTag;
-import com.sun.javadoc.Tag;
-import google.registry.model.eppoutput.Result.Code;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.TreeMap;
-import java.util.stream.Stream;
-import javax.annotation.Nullable;
-
-/**
- * Class to represent documentation information for a single EPP flow.
- *
- * <p>The static method getFlowDocs() on this class returns a list of FlowDocumentation
- * instances corresponding to the leaf flows in the flows package, constructing the instances
- * from class information returned from the javadoc system.  Each instance has methods for
- * retrieving relevant information about the flow, such as a description, error conditions, etc.
- */
-public class FlowDocumentation {
-
-  /** Constants for names of various relevant packages and classes. */
-  static final String FLOW_PACKAGE_NAME = "google.registry.flows";
-  static final String BASE_FLOW_CLASS_NAME = FLOW_PACKAGE_NAME + ".Flow";
-  static final String EXCEPTION_CLASS_NAME = FLOW_PACKAGE_NAME + ".EppException";
-  static final String CODE_ANNOTATION_NAME = EXCEPTION_CLASS_NAME + ".EppResultCode";
-
-  /** Name of the class for this flow. */
-  private final String name;
-
-  /** Fully qualified name of the class for this flow. */
-  private final String qualifiedName;
-
-  /** Name of the package in which this flow resides. */
-  private final String packageName;
-
-  /** Class docs for the flow. */
-  private final String classDocs;
-
-  /** Javadoc-tagged error conditions for this flow in list form. */
-  private final List<ErrorCase> errors;
-
-  /** Javadoc-tagged error conditions for this flow, organized by underlying error code. */
-  private final ListMultimap<Long, ErrorCase> errorsByCode;
-
-  /**
-   * Creates a FlowDocumentation for this flow class using data from javadoc tags.  Not public
-   * because clients should get FlowDocumentation objects via the DocumentationGenerator class.
-   */
-  protected FlowDocumentation(ClassDoc flowDoc) {
-    name = flowDoc.name();
-    qualifiedName = flowDoc.qualifiedName();
-    packageName = flowDoc.containingPackage().name();
-    classDocs = flowDoc.commentText();
-    errors = new ArrayList<>();
-    // Store error codes in sorted order, and leave reasons in insert order.
-    errorsByCode =
-        Multimaps.newListMultimap(new TreeMap<Long, Collection<ErrorCase>>(), ArrayList::new);
-    parseTags(flowDoc);
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public String getQualifiedName() {
-    return qualifiedName;
-  }
-
-  public String getPackageName() {
-    return packageName;
-  }
-
-  public String getClassDocs() {
-    return classDocs;
-  }
-
-  public ImmutableList<ErrorCase> getErrors() {
-    return ImmutableList.copyOf(errors);
-  }
-
-  public ImmutableMultimap<Long, ErrorCase> getErrorsByCode() {
-    return ImmutableMultimap.copyOf(errorsByCode);
-  }
-
-  /** Iterates through javadoc tags on the underlying class and calls specific parsing methods. */
-  private void parseTags(ClassDoc flowDoc) {
-    for (Tag tag : flowDoc.tags()) {
-      // Everything else is not a relevant tag.
-      if ("@error".equals(tag.name())) {
-        parseErrorTag(tag);
-      }
-    }
-  }
-
-  /** Exception to throw when an @error tag cannot be parsed correctly. */
-  private static class BadErrorTagFormatException extends IllegalStateException {
-    /** Makes a message to use as a prefix for the reason passed up to the superclass. */
-    private static String makeMessage(String reason, Tag tag) {
-      return String.format("Bad @error tag format at %s - %s", tag.position(), reason);
-    }
-
-    private BadErrorTagFormatException(String reason, Tag tag) {
-      super(makeMessage(reason, tag));
-    }
-
-    private BadErrorTagFormatException(String reason, Tag tag, Exception cause) {
-      super(makeMessage(reason, tag), cause);
-    }
-  }
-
-  /** Parses a javadoc tag corresponding to an error case and updates the error mapping. */
-  private void parseErrorTag(Tag tag) {
-    // Parse the @error tag text to find the @link inline tag.
-    SeeTag linkedTag;
-    try {
-      linkedTag =
-          Stream.of(tag.inlineTags())
-              .filter(SeeTag.class::isInstance)
-              .map(SeeTag.class::cast)
-              .collect(onlyElement());
-    } catch (NoSuchElementException | IllegalArgumentException e) {
-      throw new BadErrorTagFormatException(
-          String.format("expected one @link tag in tag text but found %s: %s",
-              (e instanceof NoSuchElementException ? "none" : "multiple"),
-              tag.text()),
-          tag, e);
-    }
-    // Check to see if the @link tag references a valid class.
-    ClassDoc exceptionRef = linkedTag.referencedClass();
-    if (exceptionRef == null) {
-      throw new BadErrorTagFormatException(
-          "could not resolve class from @link tag text: " + linkedTag.text(),
-          tag);
-    }
-    // Try to convert the referenced class into an ErrorCase; fail if it's not an EppException.
-    ErrorCase error;
-    try {
-      error = new ErrorCase(exceptionRef);
-    } catch (IllegalStateException | IllegalArgumentException e) {
-      throw new BadErrorTagFormatException(
-          "class referenced in @link is not a valid EppException: " + exceptionRef.qualifiedName(),
-          tag, e);
-    }
-    // Success; store this as a parsed error case.
-    errors.add(error);
-    errorsByCode.put(error.getCode(), error);
-  }
-
-  /**
-   * Represents an error case for a flow, with a reason for the error and the EPP error code.
-   *
-   * <p>This class is an immutable wrapper for the name of an EppException subclass that gets
-   * thrown to indicate an error condition.  It overrides equals() and hashCode() so that
-   * instances of this class can be used in collections in the normal fashion.
-   */
-  public static class ErrorCase {
-
-    /** The non-qualified name of the exception class. */
-    private final String name;
-
-    /** The fully-qualified name of the exception class. */
-    private final String className;
-
-    /** The reason this error was thrown, normally documented on the low-level exception class. */
-    private final String reason;
-
-    /** The EPP error code value corresponding to this error condition. */
-    private final long errorCode;
-
-    /** Constructs an ErrorCase from the corresponding class for a low-level flow exception. */
-    protected ErrorCase(ClassDoc exceptionDoc) {
-      name = exceptionDoc.name();
-      className = exceptionDoc.qualifiedName();
-      // The javadoc comment on the class explains the reason for the error condition.
-      reason = exceptionDoc.commentText();
-      ClassDoc highLevelExceptionDoc = getHighLevelExceptionFrom(exceptionDoc);
-      errorCode = extractErrorCode(highLevelExceptionDoc);
-      checkArgument(!exceptionDoc.isAbstract(),
-          "Cannot use an abstract subclass of EppException as an error case");
-    }
-
-    public String getName() {
-      return name;
-    }
-
-    protected String getClassName() {
-      return className;
-    }
-
-    public String getReason() {
-      return reason;
-    }
-
-    public long getCode() {
-      return errorCode;
-    }
-
-    /** Returns the direct subclass of EppException that this class is a subclass of (or is). */
-    private ClassDoc getHighLevelExceptionFrom(ClassDoc exceptionDoc) {
-      // While we're not yet at the root, move up the class hierarchy looking for EppException.
-      while (exceptionDoc.superclass() != null) {
-        if (exceptionDoc.superclass().qualifiedTypeName().equals(EXCEPTION_CLASS_NAME)) {
-          return exceptionDoc;
-        }
-        exceptionDoc = exceptionDoc.superclass();
-      }
-      // Failure; we reached the root without finding a subclass of EppException.
-      throw new IllegalArgumentException(
-          String.format("Class referenced is not a subclass of %s", EXCEPTION_CLASS_NAME));
-    }
-
-    /** Returns the corresponding EPP error code for an annotated subclass of EppException. */
-    private long extractErrorCode(ClassDoc exceptionDoc) {
-      try {
-        // We're looking for a specific annotation by name that should appear only once.
-        AnnotationDesc errorCodeAnnotation =
-            Arrays.stream(exceptionDoc.annotations())
-                .filter(
-                    anno -> anno.annotationType().qualifiedTypeName().equals(CODE_ANNOTATION_NAME))
-                .findFirst()
-                .get();
-        // The annotation should have one element whose value converts to an EppResult.Code.
-        AnnotationDesc.ElementValuePair pair = errorCodeAnnotation.elementValues()[0];
-        String enumConstant = ((FieldDoc) pair.value().value()).name();
-        return Code.valueOf(enumConstant).code;
-      } catch (IllegalStateException e) {
-        throw new IllegalStateException(
-            "No error code annotation found on exception " + exceptionDoc.name(), e);
-      } catch (ArrayIndexOutOfBoundsException | ClassCastException | IllegalArgumentException e) {
-        throw new IllegalStateException("Bad annotation on exception " + exceptionDoc.name(), e);
-      }
-    }
-
-    @Override
-    public boolean equals(@Nullable Object object) {
-      // The className field canonically identifies the EppException wrapped by this class, and
-      // all other instance state is derived from that exception, so we only check className.
-      return object instanceof ErrorCase && this.className.equals(((ErrorCase) object).className);
-    }
-
-    @Override
-    public int hashCode() {
-      // See note for equals() - only className is needed for comparisons.
-      return className.hashCode();
-    }
-  }
-}
@@ -1,159 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.documentation;
-
-import static google.registry.util.BuildPathUtils.getProjectRoot;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.flogger.FluentLogger;
-import com.google.common.io.CharStreams;
-import com.sun.javadoc.RootDoc;
-import com.sun.tools.javac.file.JavacFileManager;
-import com.sun.tools.javac.util.Context;
-import com.sun.tools.javac.util.ListBuffer;
-import com.sun.tools.javadoc.JavadocTool;
-import com.sun.tools.javadoc.Messager;
-import com.sun.tools.javadoc.ModifierFilter;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Locale;
-import javax.annotation.Nullable;
-import javax.tools.StandardLocation;
-
-/**
- * Wrapper class to simplify calls to the javadoc system and hide internal details.  An instance
- * represents a set of parameters for calling out to javadoc; these parameters can be set via
- * the appropriate methods, and determine what files and packages javadoc will process.  The
- * actual running of javadoc occurs when calling getRootDoc() to retrieve a javadoc RootDoc.
- */
-public final class JavadocWrapper {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
-  /** Shows any member visible at at least the default (package) level. */
-  private static final long VISIBILITY_MASK =
-      Modifier.PUBLIC | Modifier.PROTECTED | ModifierFilter.PACKAGE;
-
-  /** Root directory for source files. If null, will use the current directory. */
-  private static final String SOURCE_PATH = getProjectRoot().resolve("core/src/main/java")
-      .toString();
-  /** Specific source files to generate documentation for. */
-  private static final ImmutableSet<String> SOURCE_FILE_NAMES = ImmutableSet.of();
-
-  /** Specific packages to generate documentation for. */
-  private static final ImmutableSet<String> SOURCE_PACKAGE_NAMES =
-      ImmutableSet.of(FlowDocumentation.FLOW_PACKAGE_NAME);
-
-  /** Whether or not the Javadoc tool should eschew excessive log output. */
-  private static final boolean QUIET = true;
-
-  /**
-   * Obtains a Javadoc {@link RootDoc} object containing raw Javadoc documentation.
-   * Wraps a call to the static method createRootDoc() and passes in instance-specific settings.
-   */
-  public static RootDoc getRootDoc() throws IOException {
-    logger.atInfo().log("Starting Javadoc tool");
-    File sourceFilePath = new File(SOURCE_PATH);
-    logger.atInfo().log("Using source directory: %s", sourceFilePath.getAbsolutePath());
-    try {
-      return createRootDoc(
-          SOURCE_PATH,
-          SOURCE_PACKAGE_NAMES,
-          SOURCE_FILE_NAMES,
-          VISIBILITY_MASK,
-          QUIET);
-    } finally {
-      logger.atInfo().log("Javadoc tool finished");
-    }
-  }
-
-  /**
-   * Obtains a Javadoc root document object for the specified source path and package/Java names.
-   * If the source path is null, then the working directory is assumed as the source path.
-   *
-   * <p>If a list of package names is provided, then Javadoc will run on these packages and all
-   * their subpackages, based out of the specified source path.
-   *
-   * <p>If a list of file names is provided, then Javadoc will also run on these Java source files.
-   * The specified source path is not considered in this case.
-   *
-   * @see <a href="http://relation.to/12969.lace">Testing Java doclets</a>
-   * @see <a href="http://www.docjar.com/docs/api/com/sun/tools/javadoc/JavadocTool.html">JavadocTool</a>
-   */
-  private static RootDoc createRootDoc(
-      @Nullable String sourcePath,
-      Collection<String> packageNames,
-      Collection<String> fileNames,
-      long visibilityMask,
-      boolean quiet) throws IOException {
-    // Create a context to hold settings for Javadoc.
-    Context context = new Context();
-
-    // Redirect Javadoc stdout/stderr to null writers, since otherwise the Java compiler
-    // issues lots of errors for classes that are imported and visible to blaze but not
-    // visible locally to the compiler.
-    // TODO(b/19124943): Find a way to ignore those errors so we can show real ones?
-    Messager.preRegister(
-        context,
-        JavadocWrapper.class.getName(),
-        new PrintWriter(CharStreams.nullWriter()),     // For errors.
-        new PrintWriter(CharStreams.nullWriter()),     // For warnings.
-        new PrintWriter(CharStreams.nullWriter()));    // For notices.
-
-    // Set source path option for Javadoc.
-    try (JavacFileManager fileManager = new JavacFileManager(context, true, UTF_8)) {
-      List<File> sourcePathFiles = new ArrayList<>();
-      if (sourcePath != null) {
-        for (String sourcePathEntry : Splitter.on(':').split(sourcePath)) {
-          sourcePathFiles.add(new File(sourcePathEntry));
-        }
-      }
-      fileManager.setLocation(StandardLocation.SOURCE_PATH, sourcePathFiles);
-
-      // Create an instance of Javadoc.
-      JavadocTool javadocTool = JavadocTool.make0(context);
-
-      // Convert the package and file lists to a format Javadoc can understand.
-      ListBuffer<String> subPackages = new ListBuffer<>();
-      subPackages.addAll(packageNames);
-      ListBuffer<String> javaNames = new ListBuffer<>();
-      javaNames.addAll(fileNames);
-
-      // Invoke Javadoc and ask it for a RootDoc containing the specified packages.
-      return javadocTool.getRootDocImpl(
-          Locale.US.toString(),                    // Javadoc comment locale
-          UTF_8.name(),                            // Source character encoding
-          new ModifierFilter(visibilityMask),      // Element visibility filter
-          javaNames.toList(),                      // Included Java file names
-          com.sun.tools.javac.util.List.nil(),     // Doclet options
-          com.sun.tools.javac.util.List.nil(),     // Source files
-          false,                                   // Don't use BreakIterator
-          subPackages.toList(),                    // Included sub-package names
-          com.sun.tools.javac.util.List.nil(),     // Excluded package names
-          false,                                   // Read source files, not classes
-          false,                                   // Don't run legacy doclet
-          quiet);                                  // If asked, run Javadoc quietly
-    }
-  }
-
-  private JavadocWrapper() {}
-}
@@ -1,38 +0,0 @@
-#!/bin/bash
-# Copyright 2017 The Nomulus Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Generate javadoc for the project
-
-if (( $# != 3 )); then
-  echo "Usage: $0 JAVADOC ZIP OUT" 1>&2
-  exit 1
-fi
-
-JAVADOC_BINARY="$1"
-ZIP_BINARY="$2"
-TARGETFILE="$3"
-TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/generate_javadoc.XXXXXXXX")"
-PWDDIR="$(pwd)"
-
-"${JAVADOC_BINARY}" -d "${TMPDIR}" \
-  $(find java -name \*.java) \
-  -tag error:t:'EPP Errors' \
-  -subpackages google.registry \
-  -exclude google.registry.dns:google.registry.proxy:google.registry.monitoring.blackbox
-cd "${TMPDIR}"
-"${PWDDIR}/${ZIP_BINARY}" -rXoq "${PWDDIR}/${TARGETFILE}" .
-cd -
-rm -rf "${TMPDIR}"
@@ -367,6 +367,12 @@
    <url-pattern>/_dr/task/linkRdeHosts</url-pattern>
  </servlet-mapping>

+  <!-- Action to automatically re-lock a domain after unlocking it -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/relockDomain</url-pattern>
+  </servlet-mapping>
+
  <!-- Security config -->
  <security-constraint>
    <web-resource-collection>
@@ -7,7 +7,7 @@
  <sessions-enabled>true</sessions-enabled>
  <instance-class>B4_1G</instance-class>
  <manual-scaling>
-    <instances>10</instances>
+    <instances>20</instances>
  </manual-scaling>

  <system-properties>
@@ -18,7 +18,14 @@
      and streams it to cloud storage. When this job has finished successfully, it'll
      launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
    </description>
-    <schedule>every day 00:07</schedule>
+    <!--
+      This only needs to run once per day, but we launch additional jobs in case the
+      cursor is lagging behind, so it'll catch up to the current date eventually.
+
+      See <a href="../../../production/default/WEB-INF/cron.xml">production config</a> for an
+      explanation of job starting times.
+    -->
+    <schedule>every 12 hours from 00:07 to 12:07</schedule>
    <target>backend</target>
  </cron>

@@ -42,6 +42,7 @@ import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.eppcommon.AuthInfo;
@@ -255,7 +256,7 @@ public final class ResourceFlowUtils {
   * @param domain is the domain already projected at approvalTime
   */
  public static DateTime computeExDateForApprovalTime(
-      DomainBase domain, DateTime approvalTime, Period period) {
+      DomainContent domain, DateTime approvalTime, Period period) {
    boolean inAutoRenew = domain.getGracePeriodStatuses().contains(GracePeriodStatus.AUTO_RENEW);
    // inAutoRenew is set to false if the period is zero because a zero-period transfer should not
    // subsume an autorenew.
@@ -19,6 +19,7 @@ import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.model.EppResourceUtils.checkResourcesExist;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.flows.EppException;
 import google.registry.flows.ExtensionManager;
@@ -33,8 +34,6 @@ import google.registry.model.eppoutput.CheckData.ContactCheckData;
 import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.util.Clock;
-import java.util.List;
-import java.util.Set;
 import javax.inject.Inject;

 /**
@@ -59,9 +58,10 @@ public final class ContactCheckFlow implements Flow {
  public final EppResponse run() throws EppException {
    extensionManager.validate();  // There are no legal extensions for this flow.
    validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
+    ImmutableList<String> targetIds = ((Check) resourceCommand).getTargetIds();
    verifyTargetIdCount(targetIds, maxChecks);
-    Set<String> existingIds = checkResourcesExist(ContactResource.class, targetIds, clock.nowUtc());
+    ImmutableSet<String> existingIds =
+        checkResourcesExist(ContactResource.class, targetIds, clock.nowUtc());
    ImmutableList.Builder<ContactCheck> checks = new ImmutableList.Builder<>();
    for (String id : targetIds) {
      boolean unused = !existingIds.contains(id);
@@ -15,6 +15,8 @@
 package google.registry.flows.domain;

 import static com.google.common.base.Strings.emptyToNull;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.flows.domain.DomainFlowUtils.checkAllowedAccessToTld;
@@ -26,7 +28,6 @@ import static google.registry.flows.domain.DomainFlowUtils.isValidReservedCreate
 import static google.registry.flows.domain.DomainFlowUtils.validateDomainName;
 import static google.registry.flows.domain.DomainFlowUtils.validateDomainNameWithIdnTables;
 import static google.registry.flows.domain.DomainFlowUtils.verifyNotInPredelegation;
-import static google.registry.model.EppResourceUtils.checkResourcesExist;
 import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
 import static google.registry.model.registry.label.ReservationType.getTypeOfHighestSeverity;

@@ -48,11 +49,14 @@ import google.registry.flows.custom.DomainCheckFlowCustomLogic.BeforeResponsePar
 import google.registry.flows.custom.DomainCheckFlowCustomLogic.BeforeResponseReturnData;
 import google.registry.flows.domain.token.AllocationTokenDomainCheckResults;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils;
+import google.registry.model.EppResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainCommand.Check;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
 import google.registry.model.domain.fee.FeeCheckCommandExtensionItem;
 import google.registry.model.domain.fee.FeeCheckResponseExtensionItem;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
+import google.registry.model.domain.fee06.FeeCheckCommandExtensionV06;
 import google.registry.model.domain.launch.LaunchCheckExtension;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationTokenExtension;
@@ -62,14 +66,14 @@ import google.registry.model.eppoutput.CheckData.DomainCheck;
 import google.registry.model.eppoutput.CheckData.DomainCheckData;
 import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.eppoutput.EppResponse.ResponseExtension;
+import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.model.registry.label.ReservationType;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
+import google.registry.persistence.VKey;
 import google.registry.util.Clock;
 import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import javax.inject.Inject;
@@ -110,14 +114,20 @@ public final class DomainCheckFlow implements Flow {
  @Inject ExtensionManager extensionManager;
  @Inject EppInput eppInput;
  @Inject @ClientId String clientId;
-  @Inject @Config("maxChecks") int maxChecks;
+
+  @Inject
+  @Config("maxChecks")
+  int maxChecks;
+
  @Inject @Superuser boolean isSuperuser;
  @Inject Clock clock;
  @Inject EppResponse.Builder responseBuilder;
  @Inject AllocationTokenFlowUtils allocationTokenFlowUtils;
  @Inject DomainCheckFlowCustomLogic flowCustomLogic;
  @Inject DomainPricingLogic pricingLogic;
-  @Inject DomainCheckFlow() {}
+
+  @Inject
+  DomainCheckFlow() {}

  @Override
  public EppResponse run() throws EppException {
@@ -126,39 +136,41 @@ public final class DomainCheckFlow implements Flow {
    flowCustomLogic.beforeValidation();
    extensionManager.validate();
    validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
+    ImmutableList<String> domainNames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(domainNames, maxChecks);
    DateTime now = clock.nowUtc();
-    ImmutableMap.Builder<String, InternetDomainName> domains = new ImmutableMap.Builder<>();
+    ImmutableMap.Builder<String, InternetDomainName> parsedDomainsBuilder =
+        new ImmutableMap.Builder<>();
    // Only check that the registrar has access to a TLD the first time it is encountered
    Set<String> seenTlds = new HashSet<>();
-    for (String targetId : ImmutableSet.copyOf(targetIds)) {
-      InternetDomainName domainName = validateDomainName(targetId);
-      validateDomainNameWithIdnTables(domainName);
+    for (String domainName : ImmutableSet.copyOf(domainNames)) {
+      InternetDomainName parsedDomain = validateDomainName(domainName);
+      validateDomainNameWithIdnTables(parsedDomain);
      // This validation is moderately expensive, so cache the results.
-      domains.put(targetId, domainName);
-      String tld = domainName.parent().toString();
+      parsedDomainsBuilder.put(domainName, parsedDomain);
+      String tld = parsedDomain.parent().toString();
      boolean tldFirstTimeSeen = seenTlds.add(tld);
      if (tldFirstTimeSeen && !isSuperuser) {
        checkAllowedAccessToTld(clientId, tld);
        verifyNotInPredelegation(Registry.get(tld), now);
      }
    }
-    ImmutableMap<String, InternetDomainName> domainNames = domains.build();
+    ImmutableMap<String, InternetDomainName> parsedDomains = parsedDomainsBuilder.build();
    flowCustomLogic.afterValidation(
        DomainCheckFlowCustomLogic.AfterValidationParameters.newBuilder()
-            .setDomainNames(domainNames)
+            .setDomainNames(parsedDomains)
            // TODO: Use as of date from fee extension v0.12 instead of now, if specified.
            .setAsOfDate(now)
            .build());
-    Set<String> existingIds = checkResourcesExist(DomainBase.class, targetIds, now);
+    ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains =
+        ForeignKeyIndex.load(DomainBase.class, domainNames, now);
    Optional<AllocationTokenExtension> allocationTokenExtension =
        eppInput.getSingleExtension(AllocationTokenExtension.class);
    Optional<AllocationTokenDomainCheckResults> tokenDomainCheckResults =
        allocationTokenExtension.map(
            tokenExtension ->
                allocationTokenFlowUtils.checkDomainsWithToken(
-                    ImmutableList.copyOf(domainNames.values()),
+                    ImmutableList.copyOf(parsedDomains.values()),
                    tokenExtension.getAllocationToken(),
                    clientId,
                    now));
@@ -173,18 +185,18 @@ public final class DomainCheckFlow implements Flow {
            .orElse(ImmutableMap.of());
    Optional<AllocationToken> allocationToken =
        tokenDomainCheckResults.flatMap(AllocationTokenDomainCheckResults::token);
-    for (String targetId : targetIds) {
+    for (String domainName : domainNames) {
      Optional<String> message =
          getMessageForCheck(
-              domainNames.get(targetId),
-              existingIds,
+              parsedDomains.get(domainName),
+              existingDomains,
              domainCheckResults,
              tldStates,
              allocationToken);
      boolean isAvailable = !message.isPresent();
-      checksBuilder.add(DomainCheck.create(isAvailable, targetId, message.orElse(null)));
+      checksBuilder.add(DomainCheck.create(isAvailable, domainName, message.orElse(null)));
      if (isAvailable) {
-        availableDomains.add(targetId);
+        availableDomains.add(domainName);
      }
    }
    BeforeResponseReturnData responseData =
@@ -193,7 +205,11 @@ public final class DomainCheckFlow implements Flow {
                .setDomainChecks(checksBuilder.build())
                .setResponseExtensions(
                    getResponseExtensions(
-                        domainNames, availableDomains.build(), now, allocationToken))
+                        parsedDomains,
+                        existingDomains,
+                        availableDomains.build(),
+                        now,
+                        allocationToken))
                .setAsOfDate(now)
                .build());
    return responseBuilder
@@ -204,11 +220,11 @@ public final class DomainCheckFlow implements Flow {

  private Optional<String> getMessageForCheck(
      InternetDomainName domainName,
-      Set<String> existingIds,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains,
      ImmutableMap<InternetDomainName, String> tokenCheckResults,
-      Map<String, TldState> tldStates,
+      ImmutableMap<String, TldState> tldStates,
      Optional<AllocationToken> allocationToken) {
-    if (existingIds.contains(domainName.toString())) {
+    if (existingDomains.containsKey(domainName.toString())) {
      return Optional.of("In use");
    }
    TldState tldState = tldStates.get(domainName.parent().toString());
@@ -228,6 +244,7 @@ public final class DomainCheckFlow implements Flow {
  /** Handle the fee check extension. */
  private ImmutableList<? extends ResponseExtension> getResponseExtensions(
      ImmutableMap<String, InternetDomainName> domainNames,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains,
      ImmutableSet<String> availableDomains,
      DateTime now,
      Optional<AllocationToken> allocationToken)
@@ -240,6 +257,9 @@ public final class DomainCheckFlow implements Flow {
    FeeCheckCommandExtension<?, ?> feeCheck = feeCheckOpt.get();
    ImmutableList.Builder<FeeCheckResponseExtensionItem> responseItems =
        new ImmutableList.Builder<>();
+    ImmutableMap<String, EppResource> domainObjs =
+        loadDomainsForRestoreChecks(feeCheck, domainNames, existingDomains);
+
    for (FeeCheckCommandExtensionItem feeCheckItem : feeCheck.getItems()) {
      for (String domainName : getDomainNamesToCheckForFee(feeCheckItem, domainNames.keySet())) {
        FeeCheckResponseExtensionItem.Builder<?> builder = feeCheckItem.createResponseBuilder();
@@ -247,7 +267,7 @@ public final class DomainCheckFlow implements Flow {
            feeCheckItem,
            builder,
            domainNames.get(domainName),
-            Optional.empty(),
+            Optional.ofNullable((DomainBase) domainObjs.get(domainName)),
            feeCheck.getCurrency(),
            now,
            pricingLogic,
@@ -259,12 +279,57 @@ public final class DomainCheckFlow implements Flow {
    return ImmutableList.of(feeCheck.createResponse(responseItems.build()));
  }

+  /**
+   * Loads and returns all existing domains that are having restore fees checked.
+   *
+   * <p>This is necessary so that we can check their expiration dates to determine if a one-year
+   * renewal is part of the cost of a restore.
+   *
+   * <p>This may be resource-intensive for large checks of many restore fees, but those are
+   * comparatively rare, and we are at least using an in-memory cache. Also this will get a lot
+   * nicer in Cloud SQL when we can SELECT just the fields we want rather than having to load the
+   * entire entity.
+   */
+  private ImmutableMap<String, EppResource> loadDomainsForRestoreChecks(
+      FeeCheckCommandExtension<?, ?> feeCheck,
+      ImmutableMap<String, InternetDomainName> domainNames,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains) {
+    ImmutableList<String> restoreCheckDomains;
+    if (feeCheck instanceof FeeCheckCommandExtensionV06) {
+      // The V06 fee extension supports specifying the command fees to check on a per-domain basis.
+      restoreCheckDomains =
+          feeCheck.getItems().stream()
+              .filter(fc -> fc.getCommandName() == CommandName.RESTORE)
+              .map(FeeCheckCommandExtensionItem::getDomainName)
+              .collect(toImmutableList());
+    } else if (feeCheck.getItems().stream()
+        .anyMatch(fc -> fc.getCommandName() == CommandName.RESTORE)) {
+      // The more recent fee extension versions support specifying the command fees to check only on
+      // the overall domain check, not per-domain.
+      restoreCheckDomains = ImmutableList.copyOf(domainNames.keySet());
+    } else {
+      // Fall-through case for more recent fee extension versions when the restore fee isn't being
+      // checked.
+      restoreCheckDomains = ImmutableList.of();
+    }
+
+    // Filter down to just domains we know exist and then use the EppResource cache to load them.
+    ImmutableMap<String, VKey<DomainBase>> existingDomainsToLoad =
+        restoreCheckDomains.stream()
+            .filter(existingDomains::containsKey)
+            .collect(toImmutableMap(d -> d, d -> existingDomains.get(d).getResourceKey()));
+    ImmutableMap<VKey<? extends EppResource>, EppResource> loadedDomains =
+        EppResource.loadCached(ImmutableList.copyOf(existingDomainsToLoad.values()));
+    return ImmutableMap.copyOf(
+        Maps.transformEntries(existingDomainsToLoad, (k, v) -> loadedDomains.get(v)));
+  }
+
  /**
   * Return the domains to be checked for a particular fee check item. Some versions of the fee
   * extension specify the domain name in the extension item, while others use the list of domain
   * names from the regular check domain availability list.
   */
-  private Set<String> getDomainNamesToCheckForFee(
+  private ImmutableSet<String> getDomainNamesToCheckForFee(
      FeeCheckCommandExtensionItem feeCheckItem, ImmutableSet<String> availabilityCheckDomains)
      throws OnlyCheckedNamesCanBeFeeCheckedException {
    if (feeCheckItem.isDomainNameSupported()) {
@@ -48,7 +48,6 @@ import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.model.tmch.ClaimsListShard;
 import google.registry.util.Clock;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import javax.inject.Inject;
@@ -87,14 +86,14 @@ public final class DomainClaimsCheckFlow implements Flow {
    if (eppInput.getSingleExtension(AllocationTokenExtension.class).isPresent()) {
      throw new DomainClaimsCheckNotAllowedWithAllocationTokens();
    }
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
+    ImmutableList<String> domainNames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(domainNames, maxChecks);
    Set<String> seenTlds = new HashSet<>();
    ImmutableList.Builder<LaunchCheck> launchChecksBuilder = new ImmutableList.Builder<>();
-    for (String targetId : ImmutableSet.copyOf(targetIds)) {
-      InternetDomainName domainName = validateDomainName(targetId);
-      validateDomainNameWithIdnTables(domainName);
-      String tld = domainName.parent().toString();
+    for (String domainName : ImmutableSet.copyOf(domainNames)) {
+      InternetDomainName parsedDomain = validateDomainName(domainName);
+      validateDomainNameWithIdnTables(parsedDomain);
+      String tld = parsedDomain.parent().toString();
      // Only validate access to a TLD the first time it is encountered.
      if (seenTlds.add(tld)) {
        if (!isSuperuser) {
@@ -105,10 +104,10 @@ public final class DomainClaimsCheckFlow implements Flow {
          verifyClaimsPeriodNotEnded(registry, now);
        }
      }
-      Optional<String> claimKey = ClaimsListShard.get().getClaimKey(domainName.parts().get(0));
+      Optional<String> claimKey = ClaimsListShard.get().getClaimKey(parsedDomain.parts().get(0));
      launchChecksBuilder.add(
          LaunchCheck.create(
-              LaunchCheckName.create(claimKey.isPresent(), targetId), claimKey.orElse(null)));
+              LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null)));
    }
    return responseBuilder
        .setOnlyExtension(LaunchCheckResponseExtension.create(CLAIMS, launchChecksBuilder.build()))
@@ -347,8 +347,8 @@ public class DomainCreateFlow implements TransactionalFlow {
            .setRepoId(repoId)
            .setIdnTableName(validateDomainNameWithIdnTables(domainName))
            .setRegistrationExpirationTime(registrationExpirationTime)
-            .setAutorenewBillingEvent(Key.create(autorenewBillingEvent))
-            .setAutorenewPollMessage(Key.create(autorenewPollMessage))
+            .setAutorenewBillingEvent(autorenewBillingEvent.createVKey())
+            .setAutorenewPollMessage(autorenewPollMessage.createVKey())
            .setLaunchNotice(hasClaimsNotice ? launchCreate.get().getNotice() : null)
            .setSmdId(signedMarkId)
            .setDsData(secDnsCreate.isPresent() ? secDnsCreate.get().getDsData() : null)
@@ -31,7 +31,6 @@ import static google.registry.model.ResourceTransferUtils.handlePendingTransferO
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.ADD_FIELDS;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.RENEW_FIELDS;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -209,7 +208,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
      PollMessage.OneTime deletePollMessage =
          createDeletePollMessage(existingDomain, historyEntry, deletionTime);
      entitiesToSave.add(deletePollMessage);
-      builder.setDeletePollMessage(Key.create(deletePollMessage));
+      builder.setDeletePollMessage(deletePollMessage.createVKey());
    }

    // Cancel any grace periods that were still active, and set the expiration time accordingly.
@@ -222,8 +221,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
        if (gracePeriod.getOneTimeBillingEvent() != null) {
          // Take the amount of amount of registration time being refunded off the expiration time.
          // This can be either add grace periods or renew grace periods.
-          BillingEvent.OneTime oneTime =
-              ofy().load().key(gracePeriod.getOneTimeBillingEvent()).now();
+          BillingEvent.OneTime oneTime = tm().load(gracePeriod.getOneTimeBillingEvent());
          newExpirationTime = newExpirationTime.minusYears(oneTime.getPeriodYears());
        } else if (gracePeriod.getRecurringBillingEvent() != null) {
          // Take 1 year off the registration if in the autorenew grace period (no need to load the
@@ -370,12 +368,12 @@ public final class DomainDeleteFlow implements TransactionalFlow {
  private Money getGracePeriodCost(GracePeriod gracePeriod, DateTime now) {
    if (gracePeriod.getType() == GracePeriodStatus.AUTO_RENEW) {
      DateTime autoRenewTime =
-          ofy().load().key(checkNotNull(gracePeriod.getRecurringBillingEvent())).now()
+          tm().load(checkNotNull(gracePeriod.getRecurringBillingEvent()))
              .getRecurrenceTimeOfYear()
-                  .getLastInstanceBeforeOrAt(now);
+              .getLastInstanceBeforeOrAt(now);
      return getDomainRenewCost(targetId, autoRenewTime, 1);
    }
-    return ofy().load().key(checkNotNull(gracePeriod.getOneTimeBillingEvent())).now().getCost();
+    return tm().load(checkNotNull(gracePeriod.getOneTimeBillingEvent())).getCost();
  }

  @Nullable
@@ -17,6 +17,7 @@ package google.registry.flows.domain;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Predicates.equalTo;
+import static com.google.common.base.Strings.emptyToNull;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
@@ -32,7 +33,9 @@ import static google.registry.model.registry.Registry.TldState.GENERAL_AVAILABIL
 import static google.registry.model.registry.Registry.TldState.PREDELEGATION;
 import static google.registry.model.registry.Registry.TldState.QUIET_PERIOD;
 import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
+import static google.registry.model.registry.label.ReservationType.ALLOWED_IN_SUNRISE;
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
+import static google.registry.model.registry.label.ReservationType.NAME_COLLISION;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_ANCHOR_TENANT;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_SPECIFIC_USE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -87,6 +90,7 @@ import google.registry.model.domain.DomainCommand.InvalidReferencesException;
 import google.registry.model.domain.DomainCommand.Update;
 import google.registry.model.domain.ForeignKeyedDesignatedContact;
 import google.registry.model.domain.Period;
+import google.registry.model.domain.fee.BaseFee;
 import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Credit;
 import google.registry.model.domain.fee.Fee;
@@ -152,9 +156,7 @@ public class DomainFlowUtils {

  /** Reservation types that are only allowed in sunrise by policy. */
  public static final ImmutableSet<ReservationType> TYPES_ALLOWED_FOR_CREATE_ONLY_IN_SUNRISE =
-      Sets.immutableEnumSet(
-          ReservationType.ALLOWED_IN_SUNRISE,
-          ReservationType.NAME_COLLISION);
+      Sets.immutableEnumSet(ALLOWED_IN_SUNRISE, NAME_COLLISION);

  /** Warning message for allocation of collision domains in sunrise. */
  public static final String COLLISION_MESSAGE =
@@ -515,14 +517,14 @@ public class DomainFlowUtils {
   */
  public static void updateAutorenewRecurrenceEndTime(DomainBase domain, DateTime newEndTime) {
    Optional<PollMessage.Autorenew> autorenewPollMessage =
-        Optional.ofNullable(ofy().load().key(domain.getAutorenewPollMessage()).now());
+        tm().maybeLoad(domain.getAutorenewPollMessage());

    // Construct an updated autorenew poll message. If the autorenew poll message no longer exists,
    // create a new one at the same id. This can happen if a transfer was requested on a domain
    // where all autorenew poll messages had already been delivered (this would cause the poll
    // message to be deleted), and then subsequently the transfer was canceled, rejected, or deleted
    // (which would cause the poll message to be recreated here).
-    Key<PollMessage.Autorenew> existingAutorenewKey = domain.getAutorenewPollMessage();
+    Key<PollMessage.Autorenew> existingAutorenewKey = domain.getAutorenewPollMessage().getOfyKey();
    PollMessage.Autorenew updatedAutorenewPollMessage =
        autorenewPollMessage.isPresent()
            ? autorenewPollMessage.get().asBuilder().setAutorenewEndTime(newEndTime).build()
@@ -540,7 +542,7 @@ public class DomainFlowUtils {
      ofy().save().entity(updatedAutorenewPollMessage);
    }

-    Recurring recurring = ofy().load().key(domain.getAutorenewBillingEvent()).now();
+    Recurring recurring = tm().load(domain.getAutorenewBillingEvent());
    ofy().save().entity(recurring.asBuilder().setRecurrenceEndTime(newEndTime).build());
  }

@@ -583,15 +585,15 @@ public class DomainFlowUtils {
    builder
        .setCommand(feeRequest.getCommandName(), feeRequest.getPhase(), feeRequest.getSubphase())
        .setCurrencyIfSupported(registry.getCurrency())
-        .setPeriod(feeRequest.getPeriod())
-        .setClass(pricingLogic.getFeeClass(domainNameString, now).orElse(null));
+        .setPeriod(feeRequest.getPeriod());

+    String feeClass = null;
    ImmutableList<Fee> fees = ImmutableList.of();
    switch (feeRequest.getCommandName()) {
      case CREATE:
        // Don't return a create price for reserved names.
        if (isReserved(domainName, isSunrise) && !isAvailable) {
-          builder.setClass("reserved"); // Override whatever class we've set above.
+          feeClass = "reserved";
          builder.setAvailIfSupported(false);
          builder.setReasonIfSupported("reserved");
        } else {
@@ -616,15 +618,11 @@ public class DomainFlowUtils {
          throw new RestoresAreAlwaysForOneYearException();
        }
        builder.setAvailIfSupported(true);
-        // The domain object is present only on domain info commands, not on domain check commands,
-        // because check commands can query up to 50 domains and it isn't performant to load them
-        // all. So, only on info commands can we actually determine if we should include the renewal
-        // fee because the domain needs to have been loaded in order to know its expiration time. We
-        // default to including the renewal fee on domain checks because typically most domains are
-        // deleted during the autorenew grace period and thus if restored will require a renewal,
-        // but this is just a best guess.
+        // Domains that never existed, or that used to exist but have completed the entire deletion
+        // process, don't count as expired for the purposes of requiring an added year of renewal on
+        // restore because they can't be restored in the first place.
        boolean isExpired =
-            !domain.isPresent() || domain.get().getRegistrationExpirationTime().isBefore(now);
+            domain.isPresent() && domain.get().getRegistrationExpirationTime().isBefore(now);
        fees = pricingLogic.getRestorePrice(registry, domainNameString, now, isExpired).getFees();
        break;
      case TRANSFER:
@@ -642,6 +640,23 @@ public class DomainFlowUtils {
        throw new UnknownFeeCommandException(feeRequest.getUnparsedCommandName());
    }

+    if (feeClass == null) {
+      // Calculate and set the correct fee class based on whether the name is a collision name or we
+      // are returning any premium fees, but only if the fee class isn't already set (i.e. because
+      // the domain is reserved, which overrides any other classes).
+      boolean isNameCollisionInSunrise =
+          registry.getTldState(now).equals(START_DATE_SUNRISE)
+              && getReservationTypes(domainName).contains(NAME_COLLISION);
+      boolean isPremium = fees.stream().anyMatch(BaseFee::isPremium);
+      feeClass =
+          emptyToNull(
+              Joiner.on('-')
+                  .skipNulls()
+                  .join(
+                      isPremium ? "premium" : null, isNameCollisionInSunrise ? "collision" : null));
+    }
+    builder.setClass(feeClass);
+
    // Set the fees, and based on the validDateRange of the fees, set the notAfterDate.
    if (!fees.isEmpty()) {
      builder.setFees(fees);
@@ -15,7 +15,6 @@
 package google.registry.flows.domain;

 import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
-import static google.registry.pricing.PricingEngineProxy.getDomainFeeClass;
 import static google.registry.pricing.PricingEngineProxy.getPricesForDomainName;

 import com.google.common.net.InternetDomainName;
@@ -186,11 +185,6 @@ public final class DomainPricingLogic {
            .build());
  }

-  /** Returns the fee class for a given domain and date. */
-  public Optional<String> getFeeClass(String domainName, DateTime dateTime) {
-    return getDomainFeeClass(domainName, dateTime);
-  }
-
  /** Returns the domain create cost with allocation-token-related discounts applied. */
  private Money getDomainCreateCostWithDiscount(
      DomainPrices domainPrices, int years, Optional<AllocationToken> allocationToken)
@@ -33,7 +33,6 @@ import static google.registry.util.DateTimeUtils.leapSafeAddYears;

 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
 import google.registry.flows.EppException.ParameterValueRangeErrorException;
 import google.registry.flows.ExtensionManager;
@@ -181,8 +180,8 @@ public final class DomainRenewFlow implements TransactionalFlow {
            .setLastEppUpdateTime(now)
            .setLastEppUpdateClientId(clientId)
            .setRegistrationExpirationTime(newExpirationTime)
-            .setAutorenewBillingEvent(Key.create(newAutorenewEvent))
-            .setAutorenewPollMessage(Key.create(newAutorenewPollMessage))
+            .setAutorenewBillingEvent(newAutorenewEvent.createVKey())
+            .setAutorenewPollMessage(newAutorenewPollMessage.createVKey())
            .addGracePeriod(
                GracePeriod.forBillingEvent(GracePeriodStatus.RENEW, explicitRenewEvent))
            .build();
@@ -26,7 +26,6 @@ import static google.registry.flows.domain.DomainFlowUtils.verifyNotReserved;
 import static google.registry.flows.domain.DomainFlowUtils.verifyPremiumNameIsNotBlocked;
 import static google.registry.flows.domain.DomainFlowUtils.verifyRegistrarIsActive;
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;

@@ -174,8 +173,8 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
            existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId);
    updateForeignKeyIndexDeletionTime(newDomain);
    entitiesToSave.add(newDomain, historyEntry, autorenewEvent, autorenewPollMessage);
-    ofy().save().entities(entitiesToSave.build());
-    ofy().delete().key(existingDomain.getDeletePollMessage());
+    tm().saveNewOrUpdateAll(entitiesToSave.build());
+    tm().delete(existingDomain.getDeletePollMessage());
    dnsQueue.addDomainRefreshTask(existingDomain.getDomainName());
    return responseBuilder
        .setExtensions(createResponseExtensions(feesAndCredits, feeUpdate, isExpired))
@@ -232,8 +231,8 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
        .setStatusValues(null)
        .setGracePeriods(null)
        .setDeletePollMessage(null)
-        .setAutorenewBillingEvent(Key.create(autorenewEvent))
-        .setAutorenewPollMessage(Key.create(autorenewPollMessage))
+        .setAutorenewBillingEvent(autorenewEvent.createVKey())
+        .setAutorenewPollMessage(autorenewPollMessage.createVKey())
        .setLastEppUpdateTime(now)
        .setLastEppUpdateClientId(clientId)
        .build();
@@ -186,8 +186,8 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
                    .setTransferredRegistrationExpirationTime(newExpirationTime)
                    .build())
            .setRegistrationExpirationTime(newExpirationTime)
-            .setAutorenewBillingEvent(Key.create(autorenewEvent))
-            .setAutorenewPollMessage(Key.create(gainingClientAutorenewPollMessage))
+            .setAutorenewBillingEvent(autorenewEvent.createVKey())
+            .setAutorenewPollMessage(gainingClientAutorenewPollMessage.createVKey())
            // Remove all the old grace periods and add a new one for the transfer.
            .setGracePeriods(
                billingEvent.isPresent()
@@ -16,12 +16,12 @@ package google.registry.flows.domain.token;

 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import com.google.common.net.InternetDomainName;
 import google.registry.flows.EppException;
 import google.registry.model.domain.DomainCommand;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.registry.Registry;
-import java.util.function.Function;
 import org.joda.time.DateTime;

 /**
@@ -50,7 +50,6 @@ public class AllocationTokenCustomLogic {
      String clientId,
      DateTime now) {
    // Do nothing.
-    return domainNames.stream()
-        .collect(ImmutableMap.toImmutableMap(Function.identity(), ignored -> ""));
+    return Maps.toMap(domainNames, k -> "");
  }
 }
@@ -19,6 +19,7 @@ import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.model.EppResourceUtils.checkResourcesExist;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.flows.EppException;
 import google.registry.flows.ExtensionManager;
@@ -33,8 +34,6 @@ import google.registry.model.host.HostCommand.Check;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.util.Clock;
-import java.util.List;
-import java.util.Set;
 import javax.inject.Inject;

 /**
@@ -59,13 +58,14 @@ public final class HostCheckFlow implements Flow {
  public final EppResponse run() throws EppException {
    extensionManager.validate();  // There are no legal extensions for this flow.
    validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
-    Set<String> existingIds = checkResourcesExist(HostResource.class, targetIds, clock.nowUtc());
+    ImmutableList<String> hostnames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(hostnames, maxChecks);
+    ImmutableSet<String> existingIds =
+        checkResourcesExist(HostResource.class, hostnames, clock.nowUtc());
    ImmutableList.Builder<HostCheck> checks = new ImmutableList.Builder<>();
-    for (String id : targetIds) {
-      boolean unused = !existingIds.contains(id);
-      checks.add(HostCheck.create(unused, id, unused ? null : "In use"));
+    for (String hostname : hostnames) {
+      boolean unused = !existingIds.contains(hostname);
+      checks.add(HostCheck.create(unused, hostname, unused ? null : "In use"));
    }
    return responseBuilder.setResData(HostCheckData.create(checks.build())).build();
  }
@@ -14,8 +14,8 @@

 package google.registry.flows.poll;

-import static com.google.common.base.Preconditions.checkState;
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
+import static google.registry.flows.poll.PollFlowUtils.ackPollMessage;
 import static google.registry.flows.poll.PollFlowUtils.getPollMessagesQuery;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_NO_MESSAGES;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -100,27 +100,8 @@ public class PollAckFlow implements TransactionalFlow {
    // This keeps track of whether we should include the current acked message in the updated
    // message count that's returned to the user. The only case where we do so is if an autorenew
    // poll message is acked, but its next event is already ready to be delivered.
-    boolean includeAckedMessageInCount = false;
-    if (pollMessage instanceof PollMessage.OneTime) {
-      // One-time poll messages are deleted once acked.
-      ofy().delete().entity(pollMessage);
-    } else {
-      checkState(pollMessage instanceof PollMessage.Autorenew, "Unknown poll message type");
-      PollMessage.Autorenew autorenewPollMessage = (PollMessage.Autorenew) pollMessage;
+    boolean includeAckedMessageInCount = ackPollMessage(pollMessage);

-      // Move the eventTime of this autorenew poll message forward by a year.
-      DateTime nextEventTime = autorenewPollMessage.getEventTime().plusYears(1);
-
-      // If the next event falls within the bounds of the end time, then just update the eventTime
-      // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
-      // autorenew poll message has no more events to deliver and should be deleted.
-      if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
-        ofy().save().entity(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
-        includeAckedMessageInCount = isBeforeOrAt(nextEventTime, now);
-      } else {
-        ofy().delete().entity(autorenewPollMessage);
-      }
-    }
    // We need to return the new queue length. If this was the last message in the queue being
    // acked, then we return a special status code indicating that. Note that the query will
    // include the message being acked.
@@ -14,7 +14,10 @@

 package google.registry.flows.poll;

+import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;

 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.poll.PollMessage;
@@ -33,4 +36,42 @@ public final class PollFlowUtils {
        .filter("eventTime <=", now.toDate())
        .order("eventTime");
  }
+
+  /**
+   * Acknowledges the given {@link PollMessage} and returns whether we should include the current
+   * acked message in the updated message count that's returned to the user.
+   *
+   * <p>The only case where we do so is if an autorenew poll message is acked, but its next event is
+   * already ready to be delivered.
+   */
+  public static boolean ackPollMessage(PollMessage pollMessage) {
+    checkArgument(
+        isBeforeOrAt(pollMessage.getEventTime(), tm().getTransactionTime()),
+        "Cannot ACK poll message with ID %s because its event time is in the future: %s",
+        pollMessage.getId(),
+        pollMessage.getEventTime());
+    boolean includeAckedMessageInCount = false;
+    if (pollMessage instanceof PollMessage.OneTime) {
+      // One-time poll messages are deleted once acked.
+      tm().delete(pollMessage.createVKey());
+    } else if (pollMessage instanceof PollMessage.Autorenew) {
+      PollMessage.Autorenew autorenewPollMessage = (PollMessage.Autorenew) pollMessage;
+
+      // Move the eventTime of this autorenew poll message forward by a year.
+      DateTime nextEventTime = autorenewPollMessage.getEventTime().plusYears(1);
+
+      // If the next event falls within the bounds of the end time, then just update the eventTime
+      // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
+      // autorenew poll message has no more events to deliver and should be deleted.
+      if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
+        tm().saveNewOrUpdate(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
+        includeAckedMessageInCount = isBeforeOrAt(nextEventTime, tm().getTransactionTime());
+      } else {
+        tm().delete(autorenewPollMessage.createVKey());
+      }
+    } else {
+      throw new IllegalArgumentException("Unknown poll message type: " + pollMessage.getClass());
+    }
+    return includeAckedMessageInCount;
+  }
 }
@@ -17,7 +17,7 @@ package google.registry.keyring.kms;
 import google.registry.keyring.api.KeyringException;

 /** An abstraction to simplify Cloud KMS operations. */
-interface KmsConnection {
+public interface KmsConnection {

  /**
   * The maximum allowable secret size, as set by Cloud KMS.
@@ -25,11 +25,9 @@ import com.google.api.services.cloudkms.v1.model.DecryptRequest;
 import com.google.api.services.cloudkms.v1.model.EncryptRequest;
 import com.google.api.services.cloudkms.v1.model.KeyRing;
 import com.google.api.services.cloudkms.v1.model.UpdateCryptoKeyPrimaryVersionRequest;
-import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyringException;
 import google.registry.util.Retrier;
 import java.io.IOException;
-import javax.inject.Inject;

 /** The {@link KmsConnection} which talks to Cloud KMS. */
 class KmsConnectionImpl implements KmsConnection {
@@ -44,12 +42,7 @@ class KmsConnectionImpl implements KmsConnection {
  private final String projectId;
  private final Retrier retrier;

-  @Inject
-  KmsConnectionImpl(
-      @Config("cloudKmsProjectId") String projectId,
-      @Config("cloudKmsKeyRing") String kmsKeyRingName,
-      Retrier retrier,
-      CloudKMS kms) {
+  KmsConnectionImpl(String projectId, String kmsKeyRingName, Retrier retrier, CloudKMS kms) {
    this.projectId = projectId;
    this.kmsKeyRingName = kmsKeyRingName;
    this.retrier = retrier;
@@ -21,6 +21,7 @@ import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;

 import com.googlecode.objectify.Key;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeySerializer;
 import google.registry.keyring.api.Keyring;
 import google.registry.keyring.api.KeyringException;
@@ -86,7 +87,7 @@ public class KmsKeyring implements Keyring {
  private final KmsConnection kmsConnection;

  @Inject
-  KmsKeyring(KmsConnection kmsConnection) {
+  KmsKeyring(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
    this.kmsConnection = kmsConnection;
  }

@@ -24,6 +24,7 @@ import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.Keyring;
 import google.registry.util.GoogleCredentialsBundle;
+import google.registry.util.Retrier;

 /** Dagger module for Cloud KMS. */
@Module
@@ -32,9 +33,22 @@ public abstract class KmsModule {
  public static final String NAME = "KMS";

  @Provides
+  @Config("defaultKms")
  static CloudKMS provideKms(
      @DefaultCredential GoogleCredentialsBundle credentialsBundle,
      @Config("cloudKmsProjectId") String projectId) {
+    return createKms(credentialsBundle, projectId);
+  }
+
+  @Provides
+  @Config("beamKms")
+  static CloudKMS provideBeamKms(
+      @DefaultCredential GoogleCredentialsBundle credentialsBundle,
+      @Config("beamCloudKmsProjectId") String projectId) {
+    return createKms(credentialsBundle, projectId);
+  }
+
+  private static CloudKMS createKms(GoogleCredentialsBundle credentialsBundle, String projectId) {
    return new CloudKMS.Builder(
            credentialsBundle.getHttpTransport(),
            credentialsBundle.getJsonFactory(),
@@ -43,11 +57,28 @@ public abstract class KmsModule {
        .build();
  }

+  @Provides
+  @Config("defaultKmsConnection")
+  static KmsConnection provideKmsConnection(
+      @Config("cloudKmsProjectId") String projectId,
+      @Config("cloudKmsKeyRing") String keyringName,
+      Retrier retrier,
+      @Config("defaultKms") CloudKMS defaultKms) {
+    return new KmsConnectionImpl(projectId, keyringName, retrier, defaultKms);
+  }
+
+  @Provides
+  @Config("beamKmsConnection")
+  static KmsConnection provideBeamKmsConnection(
+      @Config("beamCloudKmsProjectId") String projectId,
+      @Config("beamCloudKmsKeyRing") String keyringName,
+      Retrier retrier,
+      @Config("beamKms") CloudKMS defaultKms) {
+    return new KmsConnectionImpl(projectId, keyringName, retrier, defaultKms);
+  }
+
  @Binds
  @IntoMap
  @StringKey(NAME)
  abstract Keyring provideKeyring(KmsKeyring keyring);
-
-  @Binds
-  abstract KmsConnection provideKmsConnection(KmsConnectionImpl kmsConnectionImpl);
 }
@@ -39,6 +39,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

 import com.google.common.collect.ImmutableMap;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeySerializer;
 import google.registry.keyring.kms.KmsKeyring.PrivateKeyLabel;
 import google.registry.keyring.kms.KmsKeyring.PublicKeyLabel;
@@ -64,7 +65,7 @@ public final class KmsUpdater {
  private final HashMap<String, byte[]> secretValues;

  @Inject
-  public KmsUpdater(KmsConnection kmsConnection) {
+  public KmsUpdater(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
    this.kmsConnection = kmsConnection;

    // Use LinkedHashMap to preserve insertion order on update() to simplify testing and debugging
@@ -14,16 +14,21 @@

 package google.registry.model;

+import com.google.common.annotations.VisibleForTesting;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlTransient;

 /**
 * Base class for entities that are the root of a Registry 2.0 entity group that gets enrolled in
 * commit logs for backup purposes.
 *
- * <p>The commit log system needs to preserve the ordering of closely timed mutations to entities
- * in a single entity group. We require an {@link UpdateAutoTimestamp} field on the root of a group
- * so that we can enforce strictly increasing timestamps.
+ * <p>The commit log system needs to preserve the ordering of closely timed mutations to entities in
+ * a single entity group. We require an {@link UpdateAutoTimestamp} field on the root of a group so
+ * that we can enforce strictly increasing timestamps.
 */
+@MappedSuperclass
 public abstract class BackupGroupRoot extends ImmutableObject {
  /**
   * An automatically managed timestamp of when this object was last written to Datastore.
@@ -32,10 +37,14 @@ public abstract class BackupGroupRoot extends ImmutableObject {
   * that this is updated on every save, rather than only in response to an {@code <update>} command
   */
  @XmlTransient
+  // Prevents subclasses from unexpectedly accessing as property (e.g., HostResource), which would
+  // require an unnecessary non-private setter method.
+  @Access(AccessType.FIELD)
+  @VisibleForTesting
  UpdateAutoTimestamp updateTimestamp = UpdateAutoTimestamp.create(null);

  /** Get the {@link UpdateAutoTimestamp} for this entity. */
-  public final UpdateAutoTimestamp getUpdateAutoTimestamp() {
+  public UpdateAutoTimestamp getUpdateTimestamp() {
    return updateTimestamp;
  }
 }
@@ -16,7 +16,6 @@ package google.registry.model;

 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
@@ -47,7 +46,6 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
-import java.util.stream.StreamSupport;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
@@ -357,16 +355,18 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
        @Override
        public Map<VKey<? extends EppResource>, EppResource> loadAll(
            Iterable<? extends VKey<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> loadAsMap(keys));
+          return tm().doTransactionless(() -> tm().load(keys));
        }
      };

  /**
   * A limited size, limited time cache for EPP resource entities.
   *
-   * <p>This is only used to cache contacts and hosts for the purposes of checking whether they are
-   * deleted or in pending delete during a few domain flows. Any operations on contacts and hosts
-   * directly should of course never use the cache.
+   * <p>This is used to cache contacts and hosts for the purposes of checking whether they are
+   * deleted or in pending delete during a few domain flows, and also to cache domains for the
+   * purpose of determining restore fees in domain checks. Any mutating operations directly on EPP
+   * resources should of course never use the cache as they always need perfectly up-to-date
+   * information.
   */
  @NonFinalForTesting
  private static LoadingCache<VKey<? extends EppResource>, EppResource> cacheEppResources =
@@ -395,7 +395,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
  public static ImmutableMap<VKey<? extends EppResource>, EppResource> loadCached(
      Iterable<VKey<? extends EppResource>> keys) {
    if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return loadAsMap(keys);
+      return tm().load(keys);
    }
    try {
      return cacheEppResources.getAll(keys);
@@ -423,17 +423,4 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
      throw new RuntimeException("Error loading cached EppResources", e.getCause());
    }
  }
-
-  private static ImmutableMap<VKey<? extends EppResource>, EppResource> loadAsMap(
-      Iterable<? extends VKey<? extends EppResource>> keys) {
-    return StreamSupport.stream(keys.spliterator(), false)
-        // It's possible for us to receive the same key more than once which causes
-        // the immutable map build to break with a duplicate key, so we have to ensure key
-        // uniqueness.
-        .distinct()
-        // We have to use "key -> key" here instead of the identity() function, because
-        // the latter breaks the fairly complicated generic type checking required by the
-        // caching interface.
-        .collect(toImmutableMap(key -> key, key -> tm().load(key)));
-  }
 }
@@ -23,6 +23,7 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.Result;
@@ -45,7 +46,6 @@ import google.registry.model.transfer.TransferStatus;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
 import javax.annotation.Nullable;
 import org.joda.time.DateTime;
@@ -155,7 +155,7 @@ public final class EppResourceUtils {
    // time for writes.
    return Optional.of(
        cloneProjectedAtTime(
-            resource, latestOf(now, resource.getUpdateAutoTimestamp().getTimestamp())));
+            resource, latestOf(now, resource.getUpdateTimestamp().getTimestamp())));
  }

  /**
@@ -169,7 +169,7 @@ public final class EppResourceUtils {
   * @param uniqueIds a list of ids to match
   * @param now the logical time of the check
   */
-  public static <T extends EppResource> Set<String> checkResourcesExist(
+  public static <T extends EppResource> ImmutableSet<String> checkResourcesExist(
      Class<T> clazz, List<String> uniqueIds, final DateTime now) {
    return ForeignKeyIndex.load(clazz, uniqueIds, now).keySet();
  }
@@ -298,7 +298,7 @@ public final class EppResourceUtils {
    // and returns it projected forward to exactly the desired timestamp, or null if the resource is
    // deleted at that timestamp.
    final Result<T> loadResult =
-        isAtOrAfter(timestamp, resource.getUpdateAutoTimestamp().getTimestamp())
+        isAtOrAfter(timestamp, resource.getUpdateTimestamp().getTimestamp())
            ? new ResultNow<>(resource)
            : loadMostRecentRevisionAtTime(resource, timestamp);
    return () -> {
@@ -347,7 +347,27 @@ public abstract class BillingEvent extends ImmutableObject

    @Override
    public VKey<OneTime> createVKey() {
-      return VKey.createOfy(getClass(), Key.create(this));
+      return VKey.create(
+          getClass(),
+          parent.getParent().getName() + "/" + parent.getId() + "/" + getId(),
+          Key.create(this));
+    }
+
+    public static VKey<OneTime> createVKey(Key<OneTime> key) {
+      // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary
+      // key type for the table, which is a single long integer.
+      if (key == null) {
+        return null;
+      }
+      Key parent = key.getParent();
+      Key grandparent = (parent != null) ? parent.getParent() : null;
+      String path =
+          (grandparent != null ? grandparent.getName() : "")
+              + "/"
+              + (parent != null ? parent.getId() : "")
+              + "/"
+              + key.getId();
+      return VKey.create(OneTime.class, path, key);
    }

    @Override
@@ -485,7 +505,21 @@ public abstract class BillingEvent extends ImmutableObject

    @Override
    public VKey<Recurring> createVKey() {
-      return VKey.createOfy(getClass(), Key.create(this));
+      return VKey.create(
+          getClass(),
+          parent.getParent().getName() + "/" + parent.getId() + "/" + getId(),
+          Key.create(this));
+    }
+
+    public static VKey<Recurring> createVKey(Key<Recurring> key) {
+      // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary
+      // key type for the table, which is a single long integer.
+      if (key == null) {
+        return null;
+      }
+      String path =
+          key.getParent().getParent().getName() + "/" + key.getParent().getId() + "/" + key.getId();
+      return VKey.create(Recurring.class, path, key);
    }

    @Override
@@ -596,18 +630,30 @@ public abstract class BillingEvent extends ImmutableObject
          .setParent(historyEntry);
      // Set the grace period's billing event using the appropriate Cancellation builder method.
      if (gracePeriod.getOneTimeBillingEvent() != null) {
-        builder.setOneTimeEventKey(
-            VKey.createOfy(BillingEvent.OneTime.class, gracePeriod.getOneTimeBillingEvent()));
+        builder.setOneTimeEventKey(gracePeriod.getOneTimeBillingEvent());
      } else if (gracePeriod.getRecurringBillingEvent() != null) {
-        builder.setRecurringEventKey(
-            VKey.createOfy(BillingEvent.Recurring.class, gracePeriod.getRecurringBillingEvent()));
+        builder.setRecurringEventKey(gracePeriod.getRecurringBillingEvent());
      }
      return builder.build();
    }

    @Override
    public VKey<Cancellation> createVKey() {
-      return VKey.createOfy(getClass(), Key.create(this));
+      return VKey.create(
+          getClass(),
+          parent.getParent().getName() + "/" + parent.getId() + "/" + getId(),
+          Key.create(this));
+    }
+
+    public static VKey<Cancellation> createVKey(Key<Cancellation> key) {
+      // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary
+      // key type for the table, which is a single long integer.
+      if (key == null) {
+        return null;
+      }
+      String path =
+          key.getParent().getParent().getName() + "/" + key.getParent().getId() + "/" + key.getId();
+      return VKey.create(Cancellation.class, path, key);
    }

    @Override
@@ -687,7 +733,21 @@ public abstract class BillingEvent extends ImmutableObject

    @Override
    public VKey<Modification> createVKey() {
-      return VKey.createOfy(this.getClass(), Key.create(this));
+      return VKey.create(
+          getClass(),
+          parent.getParent().getName() + "/" + parent.getId() + "/" + getId(),
+          Key.create(this));
+    }
+
+    public static VKey<Modification> createVKey(Key<Modification> key) {
+      // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary
+      // key type for the table, which is a single long integer.
+      if (key == null) {
+        return null;
+      }
+      String path =
+          key.getParent().getParent().getName() + "/" + key.getParent().getId() + "/" + key.getId();
+      return VKey.create(Modification.class, path, key);
    }

    /**
@@ -0,0 +1,399 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.contact;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
+
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.IgnoreSave;
+import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.condition.IfNull;
+import google.registry.model.EppResource;
+import google.registry.model.EppResource.ResourceWithTransferData;
+import google.registry.model.transfer.ContactTransferData;
+import google.registry.persistence.VKey;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.Embedded;
+import javax.persistence.MappedSuperclass;
+import javax.xml.bind.annotation.XmlElement;
+import org.joda.time.DateTime;
+
+/**
+ * A persistable contact resource including mutable and non-mutable fields.
+ *
+ * <p>This class deliberately does not include an {@link javax.persistence.Id} so that any
+ * foreign-keyed fields can refer to the proper parent entity's ID, whether we're storing this in
+ * the DB itself or as part of another entity
+ *
+ * @see <a href="https://tools.ietf.org/html/rfc5733">RFC 5733</a>
+ */
+@MappedSuperclass
+@Embeddable
+@Access(AccessType.FIELD)
+public class ContactBase extends EppResource implements ResourceWithTransferData {
+
+  /**
+   * Unique identifier for this contact.
+   *
+   * <p>This is only unique in the sense that for any given lifetime specified as the time range
+   * from (creationTime, deletionTime) there can only be one contact in Datastore with this id.
+   * However, there can be many contacts with the same id and non-overlapping lifetimes.
+   */
+  String contactId;
+
+  /**
+   * Localized postal info for the contact. All contained values must be representable in the 7-bit
+   * US-ASCII character set. Personal info; cleared by {@link ContactResource.Builder#wipeOut}.
+   */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_local_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_local_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_local_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_local_country_code"))
+  })
+  PostalInfo localizedPostalInfo;
+
+  /**
+   * Internationalized postal info for the contact. Personal info; cleared by {@link
+   * ContactResource.Builder#wipeOut}.
+   */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_i18n_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_i18n_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_i18n_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_i18n_country_code"))
+  })
+  PostalInfo internationalizedPostalInfo;
+
+  /**
+   * Contact name used for name searches. This is set automatically to be the internationalized
+   * postal name, or if null, the localized postal name, or if that is null as well, null. Personal
+   * info; cleared by {@link ContactResource.Builder#wipeOut}.
+   */
+  @Index String searchName;
+
+  /** Contact’s voice number. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
+  })
+  ContactPhoneNumber voice;
+
+  /** Contact’s fax number. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
+  })
+  ContactPhoneNumber fax;
+
+  /** Contact’s email address. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  String email;
+
+  /** Authorization info (aka transfer secret) of the contact. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
+    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
+  })
+  ContactAuthInfo authInfo;
+
+  /** Data about any pending or past transfers on this contact. */
+  ContactTransferData transferData;
+
+  /**
+   * The time that this resource was last transferred.
+   *
+   * <p>Can be null if the resource has never been transferred.
+   */
+  DateTime lastTransferTime;
+
+  // If any new fields are added which contain personal information, make sure they are cleared by
+  // the wipeOut() function, so that data is not kept around for deleted contacts.
+
+  /** Disclosure policy. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
+    @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
+    @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
+    @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
+    @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
+    @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
+  })
+  Disclose disclose;
+
+  @Override
+  public VKey<? extends ContactBase> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.create(ContactBase.class, getRepoId(), Key.create(this));
+  }
+
+  public String getContactId() {
+    return contactId;
+  }
+
+  public PostalInfo getLocalizedPostalInfo() {
+    return localizedPostalInfo;
+  }
+
+  public PostalInfo getInternationalizedPostalInfo() {
+    return internationalizedPostalInfo;
+  }
+
+  public String getSearchName() {
+    return searchName;
+  }
+
+  public ContactPhoneNumber getVoiceNumber() {
+    return voice;
+  }
+
+  public ContactPhoneNumber getFaxNumber() {
+    return fax;
+  }
+
+  public String getEmailAddress() {
+    return email;
+  }
+
+  public ContactAuthInfo getAuthInfo() {
+    return authInfo;
+  }
+
+  public Disclose getDisclose() {
+    return disclose;
+  }
+
+  public String getCurrentSponsorClientId() {
+    return getPersistedCurrentSponsorClientId();
+  }
+
+  @Override
+  public ContactTransferData getTransferData() {
+    return Optional.ofNullable(transferData).orElse(ContactTransferData.EMPTY);
+  }
+
+  @Override
+  public DateTime getLastTransferTime() {
+    return lastTransferTime;
+  }
+
+  @Override
+  public String getForeignKey() {
+    return contactId;
+  }
+
+  /**
+   * Postal info for the contact.
+   *
+   * <p>The XML marshalling expects the {@link PostalInfo} objects in a list, but we can't actually
+   * persist them to Datastore that way because Objectify can't handle collections of embedded
+   * objects that themselves contain collections, and there's a list of streets inside. This method
+   * transforms the persisted format to the XML format for marshalling.
+   */
+  @XmlElement(name = "postalInfo")
+  public ImmutableList<PostalInfo> getPostalInfosAsList() {
+    return Stream.of(localizedPostalInfo, internationalizedPostalInfo)
+        .filter(Objects::nonNull)
+        .collect(toImmutableList());
+  }
+
+  @Override
+  public ContactBase cloneProjectedAtTime(DateTime now) {
+    return cloneContactProjectedAtTime(this, now);
+  }
+
+  /**
+   * Clones the contact (or subclass). A separate static method so that we can pass in and return a
+   * T without the compiler complaining.
+   */
+  protected static <T extends ContactBase> T cloneContactProjectedAtTime(T contact, DateTime now) {
+    Builder builder = contact.asBuilder();
+    projectResourceOntoBuilderAtTime(contact, builder, now);
+    return (T) builder.build();
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder<>(clone(this));
+  }
+
+  /** A builder for constructing {@link ContactResource}, since it is immutable. */
+  public static class Builder<T extends ContactBase, B extends Builder<T, B>>
+      extends EppResource.Builder<T, B> implements BuilderWithTransferData<ContactTransferData, B> {
+
+    public Builder() {}
+
+    protected Builder(T instance) {
+      super(instance);
+    }
+
+    public B setContactId(String contactId) {
+      getInstance().contactId = contactId;
+      return thisCastToDerived();
+    }
+
+    public B setLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
+      checkArgument(
+          localizedPostalInfo == null
+              || PostalInfo.Type.LOCALIZED.equals(localizedPostalInfo.getType()));
+      getInstance().localizedPostalInfo = localizedPostalInfo;
+      return thisCastToDerived();
+    }
+
+    public B setInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
+      checkArgument(
+          internationalizedPostalInfo == null
+              || PostalInfo.Type.INTERNATIONALIZED.equals(internationalizedPostalInfo.getType()));
+      getInstance().internationalizedPostalInfo = internationalizedPostalInfo;
+      return thisCastToDerived();
+    }
+
+    public B overlayLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
+      return setLocalizedPostalInfo(
+          getInstance().localizedPostalInfo == null
+              ? localizedPostalInfo
+              : getInstance().localizedPostalInfo.overlay(localizedPostalInfo));
+    }
+
+    public B overlayInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
+      return setInternationalizedPostalInfo(
+          getInstance().internationalizedPostalInfo == null
+              ? internationalizedPostalInfo
+              : getInstance().internationalizedPostalInfo.overlay(internationalizedPostalInfo));
+    }
+
+    public B setVoiceNumber(ContactPhoneNumber voiceNumber) {
+      getInstance().voice = voiceNumber;
+      return thisCastToDerived();
+    }
+
+    public B setFaxNumber(ContactPhoneNumber faxNumber) {
+      getInstance().fax = faxNumber;
+      return thisCastToDerived();
+    }
+
+    public B setEmailAddress(String emailAddress) {
+      getInstance().email = emailAddress;
+      return thisCastToDerived();
+    }
+
+    public B setAuthInfo(ContactAuthInfo authInfo) {
+      getInstance().authInfo = authInfo;
+      return thisCastToDerived();
+    }
+
+    public B setDisclose(Disclose disclose) {
+      getInstance().disclose = disclose;
+      return thisCastToDerived();
+    }
+
+    @Override
+    public B setTransferData(ContactTransferData transferData) {
+      getInstance().transferData = transferData;
+      return thisCastToDerived();
+    }
+
+    @Override
+    public B setLastTransferTime(DateTime lastTransferTime) {
+      getInstance().lastTransferTime = lastTransferTime;
+      return thisCastToDerived();
+    }
+
+    /**
+     * Remove all personally identifying information about a contact.
+     *
+     * <p>This should be used when deleting a contact so that the soft-deleted entity doesn't
+     * contain information that the registrant requested to be deleted.
+     */
+    public B wipeOut() {
+      setEmailAddress(null);
+      setFaxNumber(null);
+      setInternationalizedPostalInfo(null);
+      setLocalizedPostalInfo(null);
+      setVoiceNumber(null);
+      return thisCastToDerived();
+    }
+
+    @Override
+    public T build() {
+      T instance = getInstance();
+      // If TransferData is totally empty, set it to null.
+      if (ContactTransferData.EMPTY.equals(instance.transferData)) {
+        setTransferData(null);
+      }
+      // Set the searchName using the internationalized and localized postal info names.
+      if ((instance.internationalizedPostalInfo != null)
+          && (instance.internationalizedPostalInfo.getName() != null)) {
+        instance.searchName = instance.internationalizedPostalInfo.getName();
+      } else if ((instance.localizedPostalInfo != null)
+          && (instance.localizedPostalInfo.getName() != null)) {
+        instance.searchName = instance.localizedPostalInfo.getName();
+      } else {
+        instance.searchName = null;
+      }
+      return super.build();
+    }
+  }
+}
@@ -0,0 +1,88 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.contact;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.EppResource;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+
+/**
+ * A persisted history entry representing an EPP modification to a contact.
+ *
+ * <p>In addition to the general history fields (e.g. action time, registrar ID) we also persist a
+ * copy of the contact entity at this point in time. We persist a raw {@link ContactBase} so that
+ * the foreign-keyed fields in that class can refer to this object.
+ */
+@Entity
+@javax.persistence.Table(
+    indexes = {
+      @javax.persistence.Index(columnList = "creationTime"),
+      @javax.persistence.Index(columnList = "historyRegistrarId"),
+      @javax.persistence.Index(columnList = "historyType"),
+      @javax.persistence.Index(columnList = "historyModificationTime")
+    })
+public class ContactHistory extends HistoryEntry {
+  // Store ContactBase instead of ContactResource so we don't pick up its @Id
+  ContactBase contactBase;
+
+  @Column(nullable = false)
+  VKey<ContactResource> contactRepoId;
+
+  /** The state of the {@link ContactBase} object at this point in time. */
+  public ContactBase getContactBase() {
+    return contactBase;
+  }
+
+  /** The key to the {@link ContactResource} this is based off of. */
+  public VKey<ContactResource> getContactRepoId() {
+    return contactRepoId;
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder(clone(this));
+  }
+
+  public static class Builder extends HistoryEntry.Builder<ContactHistory, ContactHistory.Builder> {
+
+    public Builder() {}
+
+    public Builder(ContactHistory instance) {
+      super(instance);
+    }
+
+    public Builder setContactBase(ContactBase contactBase) {
+      getInstance().contactBase = contactBase;
+      return this;
+    }
+
+    public Builder setContactRepoId(VKey<ContactResource> contactRepoId) {
+      getInstance().contactRepoId = contactRepoId;
+      contactRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
+      return this;
+    }
+
+    // We can remove this once all HistoryEntries are converted to History objects
+    @Override
+    public Builder setParent(Key<? extends EppResource> parent) {
+      super.setParent(parent);
+      getInstance().contactRepoId = VKey.create(ContactResource.class, parent.getName(), parent);
+      return this;
+    }
+  }
+}
@@ -14,36 +14,16 @@

 package google.registry.model.contact;

-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
-
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
-import com.googlecode.objectify.annotation.IgnoreSave;
-import com.googlecode.objectify.annotation.Index;
-import com.googlecode.objectify.condition.IfNull;
-import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
-import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
-import google.registry.model.contact.PostalInfo.Type;
-import google.registry.model.transfer.ContactTransferData;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.stream.Stream;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
-import javax.persistence.AttributeOverride;
-import javax.persistence.AttributeOverrides;
-import javax.persistence.Column;
-import javax.persistence.Embedded;
-import javax.xml.bind.annotation.XmlElement;
 import org.joda.time.DateTime;

 /**
@@ -66,142 +46,13 @@ import org.joda.time.DateTime;
@ExternalMessagingName("contact")
@WithStringVKey
@Access(AccessType.FIELD)
-public class ContactResource extends EppResource
-    implements DatastoreAndSqlEntity, ForeignKeyedEppResource, ResourceWithTransferData {
-
-  /**
-   * Unique identifier for this contact.
-   *
-   * <p>This is only unique in the sense that for any given lifetime specified as the time range
-   * from (creationTime, deletionTime) there can only be one contact in Datastore with this id.
-   * However, there can be many contacts with the same id and non-overlapping lifetimes.
-   */
-  String contactId;
-
-  /**
-   * Localized postal info for the contact. All contained values must be representable in the 7-bit
-   * US-ASCII character set. Personal info; cleared by {@link Builder#wipeOut}.
-   */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
-    @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
-    @AttributeOverride(
-        name = "address.streetLine1",
-        column = @Column(name = "addr_local_street_line1")),
-    @AttributeOverride(
-        name = "address.streetLine2",
-        column = @Column(name = "addr_local_street_line2")),
-    @AttributeOverride(
-        name = "address.streetLine3",
-        column = @Column(name = "addr_local_street_line3")),
-    @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
-    @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
-    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
-    @AttributeOverride(
-        name = "address.countryCode",
-        column = @Column(name = "addr_local_country_code"))
-  })
-  PostalInfo localizedPostalInfo;
-
-  /**
-   * Internationalized postal info for the contact. Personal info; cleared by {@link
-   * Builder#wipeOut}.
-   */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
-    @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
-    @AttributeOverride(
-        name = "address.streetLine1",
-        column = @Column(name = "addr_i18n_street_line1")),
-    @AttributeOverride(
-        name = "address.streetLine2",
-        column = @Column(name = "addr_i18n_street_line2")),
-    @AttributeOverride(
-        name = "address.streetLine3",
-        column = @Column(name = "addr_i18n_street_line3")),
-    @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
-    @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
-    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
-    @AttributeOverride(
-        name = "address.countryCode",
-        column = @Column(name = "addr_i18n_country_code"))
-  })
-  PostalInfo internationalizedPostalInfo;
-
-  /**
-   * Contact name used for name searches. This is set automatically to be the internationalized
-   * postal name, or if null, the localized postal name, or if that is null as well, null. Personal
-   * info; cleared by {@link Builder#wipeOut}.
-   */
-  @Index
-  String searchName;
-
-  /** Contact’s voice number. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
-    @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
-  })
-  ContactPhoneNumber voice;
-
-  /** Contact’s fax number. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
-    @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
-  })
-  ContactPhoneNumber fax;
-
-  /** Contact’s email address. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  String email;
-
-  /** Authorization info (aka transfer secret) of the contact. */
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
-    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
-  })
-  ContactAuthInfo authInfo;
-
-  /** Data about any pending or past transfers on this contact. */
-  ContactTransferData transferData;
-
-  /**
-   * The time that this resource was last transferred.
-   *
-   * <p>Can be null if the resource has never been transferred.
-   */
-  DateTime lastTransferTime;
-
-  // If any new fields are added which contain personal information, make sure they are cleared by
-  // the wipeOut() function, so that data is not kept around for deleted contacts.
-
-  /** Disclosure policy. */
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
-    @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
-    @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
-    @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
-    @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
-    @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
-  })
-  Disclose disclose;
+public class ContactResource extends ContactBase
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource {

  @Override
  public VKey<ContactResource> createVKey() {
    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
-    return VKey.createOfy(ContactResource.class, Key.create(this));
+    return VKey.create(ContactResource.class, getRepoId(), Key.create(this));
  }

  @Override
@@ -211,81 +62,9 @@ public class ContactResource extends EppResource
    return super.getRepoId();
  }

-  public String getContactId() {
-    return contactId;
-  }
-
-  public PostalInfo getLocalizedPostalInfo() {
-    return localizedPostalInfo;
-  }
-
-  public PostalInfo getInternationalizedPostalInfo() {
-    return internationalizedPostalInfo;
-  }
-
-  public String getSearchName() {
-    return searchName;
-  }
-
-  public ContactPhoneNumber getVoiceNumber() {
-    return voice;
-  }
-
-  public ContactPhoneNumber getFaxNumber() {
-    return fax;
-  }
-
-  public String getEmailAddress() {
-    return email;
-  }
-
-  public ContactAuthInfo getAuthInfo() {
-    return authInfo;
-  }
-
-  public Disclose getDisclose() {
-    return disclose;
-  }
-
-  public final String getCurrentSponsorClientId() {
-    return getPersistedCurrentSponsorClientId();
-  }
-
-  @Override
-  public final ContactTransferData getTransferData() {
-    return Optional.ofNullable(transferData).orElse(ContactTransferData.EMPTY);
-  }
-
-  @Override
-  public DateTime getLastTransferTime() {
-    return lastTransferTime;
-  }
-
-  @Override
-  public String getForeignKey() {
-    return contactId;
-  }
-
-  /**
-   * Postal info for the contact.
-   *
-   * <p>The XML marshalling expects the {@link PostalInfo} objects in a list, but we can't actually
-   * persist them to Datastore that way because Objectify can't handle collections of embedded
-   * objects that themselves contain collections, and there's a list of streets inside. This method
-   * transforms the persisted format to the XML format for marshalling.
-   */
-  @XmlElement(name = "postalInfo")
-  public ImmutableList<PostalInfo> getPostalInfosAsList() {
-    return Stream.of(localizedPostalInfo, internationalizedPostalInfo)
-        .filter(Objects::nonNull)
-        .collect(toImmutableList());
-  }
-
  @Override
  public ContactResource cloneProjectedAtTime(DateTime now) {
-    Builder builder = this.asBuilder();
-    projectResourceOntoBuilderAtTime(this, builder, now);
-    return builder.build();
+    return ContactBase.cloneContactProjectedAtTime(this, now);
  }

  @Override
@@ -294,116 +73,12 @@ public class ContactResource extends EppResource
  }

  /** A builder for constructing {@link ContactResource}, since it is immutable. */
-  public static class Builder extends EppResource.Builder<ContactResource, Builder>
-      implements BuilderWithTransferData<ContactTransferData, Builder> {
+  public static class Builder extends ContactBase.Builder<ContactResource, Builder> {

    public Builder() {}

    private Builder(ContactResource instance) {
      super(instance);
    }
-
-    public Builder setContactId(String contactId) {
-      getInstance().contactId = contactId;
-      return this;
-    }
-
-    public Builder setLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
-      checkArgument(localizedPostalInfo == null
-          || Type.LOCALIZED.equals(localizedPostalInfo.getType()));
-      getInstance().localizedPostalInfo = localizedPostalInfo;
-      return this;
-    }
-
-    public Builder setInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
-      checkArgument(internationalizedPostalInfo == null
-          || Type.INTERNATIONALIZED.equals(internationalizedPostalInfo.getType()));
-      getInstance().internationalizedPostalInfo = internationalizedPostalInfo;
-      return this;
-    }
-
-    public Builder overlayLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
-      return setLocalizedPostalInfo(getInstance().localizedPostalInfo == null
-          ? localizedPostalInfo
-          : getInstance().localizedPostalInfo.overlay(localizedPostalInfo));
-    }
-
-    public Builder overlayInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
-      return setInternationalizedPostalInfo(getInstance().internationalizedPostalInfo == null
-          ? internationalizedPostalInfo
-          : getInstance().internationalizedPostalInfo.overlay(internationalizedPostalInfo));
-    }
-
-    public Builder setVoiceNumber(ContactPhoneNumber voiceNumber) {
-      getInstance().voice = voiceNumber;
-      return this;
-    }
-
-    public Builder setFaxNumber(ContactPhoneNumber faxNumber) {
-      getInstance().fax = faxNumber;
-      return this;
-    }
-
-    public Builder setEmailAddress(String emailAddress) {
-      getInstance().email = emailAddress;
-      return this;
-    }
-
-    public Builder setAuthInfo(ContactAuthInfo authInfo) {
-      getInstance().authInfo = authInfo;
-      return this;
-    }
-
-    public Builder setDisclose(Disclose disclose) {
-      getInstance().disclose = disclose;
-      return this;
-    }
-
-    @Override
-    public Builder setTransferData(ContactTransferData transferData) {
-      getInstance().transferData = transferData;
-      return this;
-    }
-
-    @Override
-    public Builder setLastTransferTime(DateTime lastTransferTime) {
-      getInstance().lastTransferTime = lastTransferTime;
-      return thisCastToDerived();
-    }
-
-    /**
-     * Remove all personally identifying information about a contact.
-     *
-     * <p>This should be used when deleting a contact so that the soft-deleted entity doesn't
-     * contain information that the registrant requested to be deleted.
-     */
-    public Builder wipeOut() {
-      setEmailAddress(null);
-      setFaxNumber(null);
-      setInternationalizedPostalInfo(null);
-      setLocalizedPostalInfo(null);
-      setVoiceNumber(null);
-      return this;
-    }
-
-    @Override
-    public ContactResource build() {
-      ContactResource instance = getInstance();
-      // If TransferData is totally empty, set it to null.
-      if (ContactTransferData.EMPTY.equals(instance.transferData)) {
-        setTransferData(null);
-      }
-      // Set the searchName using the internationalized and localized postal info names.
-      if ((instance.internationalizedPostalInfo != null)
-          && (instance.internationalizedPostalInfo.getName() != null)) {
-        instance.searchName = instance.internationalizedPostalInfo.getName();
-      } else if ((instance.localizedPostalInfo != null)
-          && (instance.localizedPostalInfo.getName() != null)) {
-        instance.searchName = instance.localizedPostalInfo.getName();
-      } else {
-        instance.searchName = null;
-      }
-      return super.build();
-    }
  }
 }
@@ -85,6 +85,6 @@ public class DesignatedContact extends ImmutableObject {
  }

  public DesignatedContact reconstitute() {
-    return create(type, VKey.createOfy(ContactResource.class, contact));
+    return create(type, VKey.from(contact));
  }
 }
@@ -14,79 +14,23 @@

 package google.registry.model.domain;

-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Strings.emptyToNull;
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
-import static com.google.common.collect.Sets.difference;
-import static com.google.common.collect.Sets.intersection;
-import static com.google.common.collect.Sets.union;
-import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
-import static google.registry.model.EppResourceUtils.setAutomaticTransferSuccessProperties;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.util.CollectionUtils.forceEmptyToNull;
-import static google.registry.util.CollectionUtils.nullToEmpty;
-import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
-import static google.registry.util.CollectionUtils.union;
-import static google.registry.util.DateTimeUtils.END_OF_TIME;
-import static google.registry.util.DateTimeUtils.earliestOf;
-import static google.registry.util.DateTimeUtils.isBeforeOrAt;
-import static google.registry.util.DateTimeUtils.leapSafeAddYears;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
-import static google.registry.util.DomainNameUtils.getTldFromDomainName;
-import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
-import com.googlecode.objectify.annotation.Ignore;
-import com.googlecode.objectify.annotation.IgnoreSave;
-import com.googlecode.objectify.annotation.Index;
-import com.googlecode.objectify.annotation.OnLoad;
-import com.googlecode.objectify.condition.IfNull;
-import google.registry.flows.ResourceFlowUtils;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
-import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
-import google.registry.model.billing.BillingEvent;
-import google.registry.model.contact.ContactResource;
-import google.registry.model.domain.DesignatedContact.Type;
-import google.registry.model.domain.launch.LaunchNotice;
-import google.registry.model.domain.rgp.GracePeriodStatus;
-import google.registry.model.domain.secdns.DelegationSignerData;
-import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
-import google.registry.model.poll.PollMessage;
-import google.registry.model.registry.Registry;
-import google.registry.model.transfer.DomainTransferData;
-import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
-import google.registry.util.CollectionUtils;
-import java.util.HashSet;
-import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
-import java.util.function.Predicate;
-import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
-import javax.persistence.AttributeOverride;
-import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.ElementCollection;
-import javax.persistence.Embedded;
 import javax.persistence.JoinTable;
-import javax.persistence.PostLoad;
-import javax.persistence.Transient;
 import org.joda.time.DateTime;
-import org.joda.time.Interval;

 /**
 * A persistable domain resource including mutable and non-mutable fields.
@@ -112,194 +56,8 @@ import org.joda.time.Interval;
@WithStringVKey
@ExternalMessagingName("domain")
@Access(AccessType.FIELD)
-public class DomainBase extends EppResource
-    implements DatastoreAndSqlEntity,
-        ForeignKeyedEppResource,
-        ResourceWithTransferData<DomainTransferData> {
-
-  /** The max number of years that a domain can be registered for, as set by ICANN policy. */
-  public static final int MAX_REGISTRATION_YEARS = 10;
-
-  /** Status values which prohibit DNS information from being published. */
-  private static final ImmutableSet<StatusValue> DNS_PUBLISHING_PROHIBITED_STATUSES =
-      ImmutableSet.of(
-          StatusValue.CLIENT_HOLD,
-          StatusValue.INACTIVE,
-          StatusValue.PENDING_DELETE,
-          StatusValue.SERVER_HOLD);
-
-  /**
-   * Fully qualified domain name (puny-coded), which serves as the foreign key for this domain.
-   *
-   * <p>This is only unique in the sense that for any given lifetime specified as the time range
-   * from (creationTime, deletionTime) there can only be one domain in Datastore with this name.
-   * However, there can be many domains with the same name and non-overlapping lifetimes.
-   *
-   * @invariant fullyQualifiedDomainName == fullyQualifiedDomainName.toLowerCase(Locale.ENGLISH)
-   */
-  // TODO(b/158858642): Rename this to domainName when we are off Datastore
-  @Column(name = "domainName")
-  @Index
-  String fullyQualifiedDomainName;
-
-  /** The top level domain this is under, dernormalized from {@link #fullyQualifiedDomainName}. */
-  @Index
-  String tld;
-
-  /** References to hosts that are the nameservers for the domain. */
-  @Index
-  @ElementCollection
-  @JoinTable(name = "DomainHost")
-  Set<VKey<HostResource>> nsHosts;
-
-  /**
-   * The union of the contacts visible via {@link #getContacts} and {@link #getRegistrant}.
-   *
-   * <p>These are stored in one field so that we can query across all contacts at once.
-   */
-  @Transient Set<DesignatedContact> allContacts;
-
-  /**
-   * Contacts as they are stored in cloud SQL.
-   *
-   * <p>This information is duplicated in allContacts, and must be kept in sync with it.
-   */
-  @Ignore VKey<ContactResource> adminContact;
-
-  @Ignore VKey<ContactResource> billingContact;
-  @Ignore VKey<ContactResource> techContact;
-  @Ignore VKey<ContactResource> registrantContact;
-
-  /** Authorization info (aka transfer secret) of the domain. */
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
-    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
-  })
-  DomainAuthInfo authInfo;
-
-  /**
-   * Data used to construct DS records for this domain.
-   *
-   * <p>This is {@literal @}XmlTransient because it needs to be returned under the "extension" tag
-   * of an info response rather than inside the "infData" tag.
-   */
-  @Transient Set<DelegationSignerData> dsData;
-
-  /**
-   * The claims notice supplied when this application or domain was created, if there was one. It's
-   * {@literal @}XmlTransient because it's not returned in an info response.
-   */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "noticeId.tcnId", column = @Column(name = "launch_notice_tcn_id")),
-    @AttributeOverride(
-        name = "noticeId.validatorId",
-        column = @Column(name = "launch_notice_validator_id")),
-    @AttributeOverride(
-        name = "expirationTime",
-        column = @Column(name = "launch_notice_expiration_time")),
-    @AttributeOverride(
-        name = "acceptedTime",
-        column = @Column(name = "launch_notice_accepted_time")),
-  })
-  LaunchNotice launchNotice;
-
-  /**
-   * Name of first IDN table associated with TLD that matched the characters in this domain label.
-   *
-   * @see google.registry.tldconfig.idn.IdnLabelValidator#findValidIdnTableForTld
-   */
-  @IgnoreSave(IfNull.class)
-  String idnTableName;
-
-  /** Fully qualified host names of this domain's active subordinate hosts. */
-  Set<String> subordinateHosts;
-
-  /** When this domain's registration will expire. */
-  DateTime registrationExpirationTime;
-
-  /**
-   * The poll message associated with this domain being deleted.
-   *
-   * <p>This field should be null if the domain is not in pending delete. If it is, the field should
-   * refer to a {@link PollMessage} timed to when the domain is fully deleted. If the domain is
-   * restored, the message should be deleted.
-   */
-  @Transient Key<PollMessage.OneTime> deletePollMessage;
-
-  /**
-   * The recurring billing event associated with this domain's autorenewals.
-   *
-   * <p>The recurrence should be open ended unless the domain is in pending delete or fully deleted,
-   * in which case it should be closed at the time the delete was requested. Whenever the domain's
-   * {@link #registrationExpirationTime} is changed the recurrence should be closed, a new one
-   * should be created, and this field should be updated to point to the new one.
-   */
-  @Transient Key<BillingEvent.Recurring> autorenewBillingEvent;
-
-  /**
-   * The recurring poll message associated with this domain's autorenewals.
-   *
-   * <p>The recurrence should be open ended unless the domain is in pending delete or fully deleted,
-   * in which case it should be closed at the time the delete was requested. Whenever the domain's
-   * {@link #registrationExpirationTime} is changed the recurrence should be closed, a new one
-   * should be created, and this field should be updated to point to the new one.
-   */
-  @Transient Key<PollMessage.Autorenew> autorenewPollMessage;
-
-  /** The unexpired grace periods for this domain (some of which may not be active yet). */
-  @Transient @ElementCollection Set<GracePeriod> gracePeriods;
-
-  /**
-   * The id of the signed mark that was used to create this domain in sunrise.
-   *
-   * <p>Will only be populated for domains created in sunrise.
-   */
-  @IgnoreSave(IfNull.class)
-  String smdId;
-
-  /** Data about any pending or past transfers on this domain. */
-  DomainTransferData transferData;
-
-  /**
-   * The time that this resource was last transferred.
-   *
-   * <p>Can be null if the resource has never been transferred.
-   */
-  DateTime lastTransferTime;
-
-  @OnLoad
-  void load() {
-    // Reconstitute all of the contacts so that they have VKeys.
-    allContacts =
-        allContacts.stream().map(contact -> contact.reconstitute()).collect(toImmutableSet());
-    setContactFields(allContacts, true);
-  }
-
-  @PostLoad
-  void postLoad() {
-    // Reconstitute the contact list.
-    ImmutableSet.Builder<DesignatedContact> contactsBuilder =
-        new ImmutableSet.Builder<DesignatedContact>();
-
-    if (registrantContact != null) {
-      contactsBuilder.add(
-          DesignatedContact.create(DesignatedContact.Type.REGISTRANT, registrantContact));
-    }
-    if (billingContact != null) {
-      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.BILLING, billingContact));
-    }
-    if (techContact != null) {
-      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.TECH, techContact));
-    }
-    if (adminContact != null) {
-      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.ADMIN, adminContact));
-    }
-
-    allContacts = contactsBuilder.build();
-  }
+public class DomainBase extends DomainContent
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource {

  @Override
  @javax.persistence.Id
@@ -308,334 +66,12 @@ public class DomainBase extends EppResource
    return super.getRepoId();
  }

-  public ImmutableSet<String> getSubordinateHosts() {
-    return nullToEmptyImmutableCopy(subordinateHosts);
-  }
-
-  public DateTime getRegistrationExpirationTime() {
-    return registrationExpirationTime;
-  }
-
-  public Key<PollMessage.OneTime> getDeletePollMessage() {
-    return deletePollMessage;
-  }
-
-  public Key<BillingEvent.Recurring> getAutorenewBillingEvent() {
-    return autorenewBillingEvent;
-  }
-
-  public Key<PollMessage.Autorenew> getAutorenewPollMessage() {
-    return autorenewPollMessage;
-  }
-
-  public ImmutableSet<GracePeriod> getGracePeriods() {
-    return nullToEmptyImmutableCopy(gracePeriods);
-  }
-
-  public String getSmdId() {
-    return smdId;
-  }
-
-  @Override
-  public DomainTransferData getTransferData() {
-    return Optional.ofNullable(transferData).orElse(DomainTransferData.EMPTY);
-  }
-
-  @Override
-  public DateTime getLastTransferTime() {
-    return lastTransferTime;
-  }
-
-  @Override
-  public String getForeignKey() {
-    return fullyQualifiedDomainName;
-  }
-
-  public String getDomainName() {
-    return fullyQualifiedDomainName;
-  }
-
-  public ImmutableSet<DelegationSignerData> getDsData() {
-    return nullToEmptyImmutableCopy(dsData);
-  }
-
-  public LaunchNotice getLaunchNotice() {
-    return launchNotice;
-  }
-
-  public String getIdnTableName() {
-    return idnTableName;
-  }
-
-  public ImmutableSet<VKey<HostResource>> getNameservers() {
-    return nullToEmptyImmutableCopy(nsHosts);
-  }
-
-  public final String getCurrentSponsorClientId() {
-    return getPersistedCurrentSponsorClientId();
-  }
-
-  /** Returns true if DNS information should be published for the given domain. */
-  public boolean shouldPublishToDns() {
-    return intersection(getStatusValues(), DNS_PUBLISHING_PROHIBITED_STATUSES).isEmpty();
-  }
-
-  /**
-   * Returns the Registry Grace Period Statuses for this domain.
-   *
-   * <p>This collects all statuses from the domain's {@link GracePeriod} entries and also adds the
-   * PENDING_DELETE status if needed.
-   */
-  public ImmutableSet<GracePeriodStatus> getGracePeriodStatuses() {
-    Set<GracePeriodStatus> gracePeriodStatuses = new HashSet<>();
-    for (GracePeriod gracePeriod : getGracePeriods()) {
-      gracePeriodStatuses.add(gracePeriod.getType());
-    }
-    if (getStatusValues().contains(StatusValue.PENDING_DELETE)
-        && !gracePeriodStatuses.contains(GracePeriodStatus.REDEMPTION)) {
-      gracePeriodStatuses.add(GracePeriodStatus.PENDING_DELETE);
-    }
-    return ImmutableSet.copyOf(gracePeriodStatuses);
-  }
-
-  /** Returns the subset of grace periods having the specified type. */
-  public ImmutableSet<GracePeriod> getGracePeriodsOfType(GracePeriodStatus gracePeriodType) {
-    ImmutableSet.Builder<GracePeriod> builder = new ImmutableSet.Builder<>();
-    for (GracePeriod gracePeriod : getGracePeriods()) {
-      if (gracePeriod.getType() == gracePeriodType) {
-        builder.add(gracePeriod);
-      }
-    }
-    return builder.build();
-  }
-
-  /**
-   * The logic in this method, which handles implicit server approval of transfers, very closely
-   * parallels the logic in {@code DomainTransferApproveFlow} which handles explicit client
-   * approvals.
-   */
-  @Override
-  public DomainBase cloneProjectedAtTime(final DateTime now) {
-
-    DomainTransferData transferData = getTransferData();
-    DateTime transferExpirationTime = transferData.getPendingTransferExpirationTime();
-
-    // If there's a pending transfer that has expired, handle it.
-    if (TransferStatus.PENDING.equals(transferData.getTransferStatus())
-        && isBeforeOrAt(transferExpirationTime, now)) {
-      // Project until just before the transfer time. This will handle the case of an autorenew
-      // before the transfer was even requested or during the request period.
-      // If the transfer time is precisely the moment that the domain expires, there will not be an
-      // autorenew billing event (since we end the recurrence at transfer time and recurrences are
-      // exclusive of their ending), and we can just proceed with the transfer.
-      DomainBase domainAtTransferTime =
-          cloneProjectedAtTime(transferExpirationTime.minusMillis(1));
-
-      DateTime expirationDate = transferData.getTransferredRegistrationExpirationTime();
-      if (expirationDate == null) {
-        // Extend the registration by the correct number of years from the expiration time
-        // that was current on the domain right before the transfer, capped at 10 years from
-        // the moment of the transfer.
-        expirationDate =
-            ResourceFlowUtils.computeExDateForApprovalTime(
-                domainAtTransferTime, transferExpirationTime, transferData.getTransferPeriod());
-      }
-      // If we are within an autorenew grace period, the transfer will subsume the autorenew. There
-      // will already be a cancellation written in advance by the transfer request flow, so we don't
-      // need to worry about billing, but we do need to cancel out the expiration time increase.
-      // The transfer period saved in the transfer data will be one year, unless the superuser
-      // extension set the transfer period to zero.
-      // Set the expiration, autorenew events, and grace period for the transfer. (Transfer ends
-      // all other graces).
-      Builder builder =
-          domainAtTransferTime
-              .asBuilder()
-              .setRegistrationExpirationTime(expirationDate)
-              // Set the speculatively-written new autorenew events as the domain's autorenew
-              // events.
-              .setAutorenewBillingEvent(
-                  transferData.getServerApproveAutorenewEvent() == null
-                      ? null
-                      : transferData.getServerApproveAutorenewEvent().getOfyKey())
-              .setAutorenewPollMessage(
-                  transferData.getServerApproveAutorenewPollMessage() == null
-                      ? null
-                      : transferData.getServerApproveAutorenewPollMessage().getOfyKey());
-      if (transferData.getTransferPeriod().getValue() == 1) {
-        // Set the grace period using a key to the prescheduled transfer billing event.  Not using
-        // GracePeriod.forBillingEvent() here in order to avoid the actual Datastore fetch.
-        builder.setGracePeriods(
-            ImmutableSet.of(
-                GracePeriod.create(
-                    GracePeriodStatus.TRANSFER,
-                    transferExpirationTime.plus(
-                        Registry.get(getTld()).getTransferGracePeriodLength()),
-                    transferData.getGainingClientId(),
-                    transferData.getServerApproveBillingEvent() == null
-                        ? null
-                        : transferData.getServerApproveBillingEvent().getOfyKey())));
-      } else {
-        // There won't be a billing event, so we don't need a grace period
-        builder.setGracePeriods(ImmutableSet.of());
-      }
-      // Set all remaining transfer properties.
-      setAutomaticTransferSuccessProperties(builder, transferData);
-      builder
-          .setLastEppUpdateTime(transferExpirationTime)
-          .setLastEppUpdateClientId(transferData.getGainingClientId());
-      // Finish projecting to now.
-      return builder.build().cloneProjectedAtTime(now);
-    }
-
-    Optional<DateTime> newLastEppUpdateTime = Optional.empty();
-
-    // There is no transfer. Do any necessary autorenews for active domains.
-
-    Builder builder = asBuilder();
-    if (isBeforeOrAt(registrationExpirationTime, now) && END_OF_TIME.equals(getDeletionTime())) {
-      // Autorenew by the number of years between the old expiration time and now.
-      DateTime lastAutorenewTime = leapSafeAddYears(
-          registrationExpirationTime,
-          new Interval(registrationExpirationTime, now).toPeriod().getYears());
-      DateTime newExpirationTime  = lastAutorenewTime.plusYears(1);
-      builder
-          .setRegistrationExpirationTime(newExpirationTime)
-          .addGracePeriod(
-              GracePeriod.createForRecurring(
-                  GracePeriodStatus.AUTO_RENEW,
-                  lastAutorenewTime.plus(Registry.get(getTld()).getAutoRenewGracePeriodLength()),
-                  getCurrentSponsorClientId(),
-                  autorenewBillingEvent));
-      newLastEppUpdateTime = Optional.of(lastAutorenewTime);
-    }
-
-    // Remove any grace periods that have expired.
-    DomainBase almostBuilt = builder.build();
-    builder = almostBuilt.asBuilder();
-    for (GracePeriod gracePeriod : almostBuilt.getGracePeriods()) {
-      if (isBeforeOrAt(gracePeriod.getExpirationTime(), now)) {
-        builder.removeGracePeriod(gracePeriod);
-        if (!newLastEppUpdateTime.isPresent()
-            || isBeforeOrAt(newLastEppUpdateTime.get(), gracePeriod.getExpirationTime())) {
-          newLastEppUpdateTime = Optional.of(gracePeriod.getExpirationTime());
-        }
-      }
-    }
-
-    // It is possible that the lastEppUpdateClientId is different from current sponsor client
-    // id, so we have to do the comparison instead of having one variable just storing the most
-    // recent time.
-    if (newLastEppUpdateTime.isPresent()) {
-      if (getLastEppUpdateTime() == null
-          || newLastEppUpdateTime.get().isAfter(getLastEppUpdateTime())) {
-        builder
-            .setLastEppUpdateTime(newLastEppUpdateTime.get())
-            .setLastEppUpdateClientId(getCurrentSponsorClientId());
-      }
-    }
-
-    // Handle common properties like setting or unsetting linked status. This also handles the
-    // general case of pending transfers for other resource types, but since we've always handled
-    // a pending transfer by this point that's a no-op for domains.
-    projectResourceOntoBuilderAtTime(almostBuilt, builder, now);
-    return builder.build();
-  }
-
-  /** Return what the expiration time would be if the given number of years were added to it. */
-  public static DateTime extendRegistrationWithCap(
-      DateTime now,
-      DateTime currentExpirationTime,
-      @Nullable Integer extendedRegistrationYears) {
-    // We must cap registration at the max years (aka 10), even if that truncates the last year.
-    return earliestOf(
-        leapSafeAddYears(
-            currentExpirationTime,
-            Optional.ofNullable(extendedRegistrationYears).orElse(0)),
-        leapSafeAddYears(now, MAX_REGISTRATION_YEARS));
-  }
-
-  /** Loads and returns the fully qualified host names of all linked nameservers. */
-  public ImmutableSortedSet<String> loadNameserverHostNames() {
-    return ofy()
-        .load()
-        .keys(getNameservers().stream().map(VKey::getOfyKey).collect(toImmutableSet()))
-        .values()
-        .stream()
-        .map(HostResource::getHostName)
-        .collect(toImmutableSortedSet(Ordering.natural()));
-  }
-
-  /** A key to the registrant who registered this domain. */
-  public VKey<ContactResource> getRegistrant() {
-    return registrantContact;
-  }
-
-  public VKey<ContactResource> getAdminContact() {
-    return adminContact;
-  }
-
-  public VKey<ContactResource> getBillingContact() {
-    return billingContact;
-  }
-
-  public VKey<ContactResource> getTechContact() {
-    return techContact;
-  }
-
-  /** Associated contacts for the domain (other than registrant). */
-  public ImmutableSet<DesignatedContact> getContacts() {
-    return nullToEmpty(allContacts)
-        .stream()
-        .filter(IS_REGISTRANT.negate())
-        .collect(toImmutableSet());
-  }
-
-  public DomainAuthInfo getAuthInfo() {
-    return authInfo;
-  }
-
-  /** Returns all referenced contacts from this domain or application. */
-  public ImmutableSet<VKey<ContactResource>> getReferencedContacts() {
-    return nullToEmptyImmutableCopy(allContacts)
-        .stream()
-        .map(DesignatedContact::getContactKey)
-        .filter(Objects::nonNull)
-        .collect(toImmutableSet());
-  }
-
-  public String getTld() {
-    return tld;
-  }
-
-  /**
-   * Sets the individual contact fields from {@code contacts}.
-   *
-   * <p>The registrant field is only set if {@code includeRegistrant} is true, as this field needs
-   * to be set in some circumstances but not in others.
-   */
-  private void setContactFields(Set<DesignatedContact> contacts, boolean includeRegistrant) {
-
-    // Set the individual contact fields.
-    for (DesignatedContact contact : contacts) {
-      switch (contact.getType()) {
-        case BILLING:
-          billingContact = contact.getContactKey();
-          break;
-        case TECH:
-          techContact = contact.getContactKey();
-          break;
-        case ADMIN:
-          adminContact = contact.getContactKey();
-          break;
-        case REGISTRANT:
-          if (includeRegistrant) {
-            registrantContact = contact.getContactKey();
-          }
-          break;
-        default:
-          throw new IllegalArgumentException("Unknown contact resource type: " + contact.getType());
-      }
-    }
+  @ElementCollection
+  @JoinTable(name = "DomainHost")
+  @Access(AccessType.PROPERTY)
+  @Column(name = "host_repo_id")
+  public Set<VKey<HostResource>> getNsHosts() {
+    return super.nsHosts;
  }

  @Override
@@ -643,14 +79,15 @@ public class DomainBase extends EppResource
    return VKey.create(DomainBase.class, getRepoId(), Key.create(this));
  }

+  @Override
+  public DomainBase cloneProjectedAtTime(final DateTime now) {
+    return cloneDomainProjectedAtTime(this, now);
+  }
+
  public static VKey<DomainBase> createVKey(Key key) {
    return VKey.create(DomainBase.class, key.getName(), key);
  }

-  /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
-  private static final Predicate<DesignatedContact> IS_REGISTRANT =
-      (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);
-
  /** An override of {@link EppResource#asBuilder} with tighter typing. */
  @Override
  public Builder asBuilder() {
@@ -658,201 +95,12 @@ public class DomainBase extends EppResource
  }

  /** A builder for constructing {@link DomainBase}, since it is immutable. */
-  public static class Builder extends EppResource.Builder<DomainBase, Builder>
-      implements BuilderWithTransferData<DomainTransferData, Builder> {
+  public static class Builder extends DomainContent.Builder<DomainBase, Builder> {

    public Builder() {}

    Builder(DomainBase instance) {
      super(instance);
    }
-
-    @Override
-    public DomainBase build() {
-      DomainBase instance = getInstance();
-      // If TransferData is totally empty, set it to null.
-      if (DomainTransferData.EMPTY.equals(getInstance().transferData)) {
-        setTransferData(null);
-      }
-      // A DomainBase has status INACTIVE if there are no nameservers.
-      if (getInstance().getNameservers().isEmpty()) {
-        addStatusValue(StatusValue.INACTIVE);
-      } else {  // There are nameservers, so make sure INACTIVE isn't there.
-        removeStatusValue(StatusValue.INACTIVE);
-      }
-
-      checkArgumentNotNull(emptyToNull(instance.fullyQualifiedDomainName), "Missing domainName");
-      if (instance.getRegistrant() == null
-          && instance.allContacts.stream().anyMatch(IS_REGISTRANT)) {
-        throw new IllegalArgumentException("registrant is null but is in allContacts");
-      }
-      checkArgumentNotNull(instance.getRegistrant(), "Missing registrant");
-      instance.tld = getTldFromDomainName(instance.fullyQualifiedDomainName);
-      return super.build();
-    }
-
-    public Builder setDomainName(String domainName) {
-      checkArgument(
-          domainName.equals(canonicalizeDomainName(domainName)),
-          "Domain name must be in puny-coded, lower-case form");
-      getInstance().fullyQualifiedDomainName = domainName;
-      return thisCastToDerived();
-    }
-
-    public Builder setDsData(ImmutableSet<DelegationSignerData> dsData) {
-      getInstance().dsData = dsData;
-      return thisCastToDerived();
-    }
-
-    public Builder setRegistrant(VKey<ContactResource> registrant) {
-      // Replace the registrant contact inside allContacts.
-      getInstance().allContacts = union(
-          getInstance().getContacts(),
-          DesignatedContact.create(Type.REGISTRANT, checkArgumentNotNull(registrant)));
-
-      // Set the registrant field specifically.
-      getInstance().registrantContact = registrant;
-      return thisCastToDerived();
-    }
-
-    public Builder setAuthInfo(DomainAuthInfo authInfo) {
-      getInstance().authInfo = authInfo;
-      return thisCastToDerived();
-    }
-
-    public Builder setNameservers(VKey<HostResource> nameserver) {
-      getInstance().nsHosts = ImmutableSet.of(nameserver);
-      return thisCastToDerived();
-    }
-
-    public Builder setNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
-      getInstance().nsHosts = forceEmptyToNull(nameservers);
-      return thisCastToDerived();
-    }
-
-    public Builder addNameserver(VKey<HostResource> nameserver) {
-      return addNameservers(ImmutableSet.of(nameserver));
-    }
-
-    public Builder addNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
-      return setNameservers(
-          ImmutableSet.copyOf(union(getInstance().getNameservers(), nameservers)));
-    }
-
-    public Builder removeNameserver(VKey<HostResource> nameserver) {
-      return removeNameservers(ImmutableSet.of(nameserver));
-    }
-
-    public Builder removeNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
-      return setNameservers(
-          ImmutableSet.copyOf(difference(getInstance().getNameservers(), nameservers)));
-    }
-
-    public Builder setContacts(DesignatedContact contact) {
-      return setContacts(ImmutableSet.of(contact));
-    }
-
-    public Builder setContacts(ImmutableSet<DesignatedContact> contacts) {
-      checkArgument(contacts.stream().noneMatch(IS_REGISTRANT), "Registrant cannot be a contact");
-
-      // Replace the non-registrant contacts inside allContacts.
-      getInstance().allContacts =
-          Streams.concat(
-                  nullToEmpty(getInstance().allContacts).stream().filter(IS_REGISTRANT),
-                  contacts.stream())
-              .collect(toImmutableSet());
-
-      // Set the individual fields.
-      getInstance().setContactFields(contacts, false);
-      return thisCastToDerived();
-    }
-
-    public Builder addContacts(ImmutableSet<DesignatedContact> contacts) {
-      return setContacts(ImmutableSet.copyOf(union(getInstance().getContacts(), contacts)));
-    }
-
-    public Builder removeContacts(ImmutableSet<DesignatedContact> contacts) {
-      return setContacts(ImmutableSet.copyOf(difference(getInstance().getContacts(), contacts)));
-    }
-
-    public Builder setLaunchNotice(LaunchNotice launchNotice) {
-      getInstance().launchNotice = launchNotice;
-      return thisCastToDerived();
-    }
-
-    public Builder setIdnTableName(String idnTableName) {
-      getInstance().idnTableName = idnTableName;
-      return thisCastToDerived();
-    }
-
-    public Builder setSubordinateHosts(ImmutableSet<String> subordinateHosts) {
-      getInstance().subordinateHosts = subordinateHosts;
-      return thisCastToDerived();
-    }
-
-    public Builder addSubordinateHost(String hostToAdd) {
-      return setSubordinateHosts(ImmutableSet.copyOf(
-          union(getInstance().getSubordinateHosts(), hostToAdd)));
-    }
-
-    public Builder removeSubordinateHost(String hostToRemove) {
-      return setSubordinateHosts(ImmutableSet.copyOf(
-          CollectionUtils.difference(getInstance().getSubordinateHosts(), hostToRemove)));
-    }
-
-    public Builder setRegistrationExpirationTime(DateTime registrationExpirationTime) {
-      getInstance().registrationExpirationTime = registrationExpirationTime;
-      return this;
-    }
-
-    public Builder setDeletePollMessage(Key<PollMessage.OneTime> deletePollMessage) {
-      getInstance().deletePollMessage = deletePollMessage;
-      return this;
-    }
-
-    public Builder setAutorenewBillingEvent(
-        Key<BillingEvent.Recurring> autorenewBillingEvent) {
-      getInstance().autorenewBillingEvent = autorenewBillingEvent;
-      return this;
-    }
-
-    public Builder setAutorenewPollMessage(
-        Key<PollMessage.Autorenew> autorenewPollMessage) {
-      getInstance().autorenewPollMessage = autorenewPollMessage;
-      return this;
-    }
-
-    public Builder setSmdId(String smdId) {
-      getInstance().smdId = smdId;
-      return this;
-    }
-
-    public Builder setGracePeriods(ImmutableSet<GracePeriod> gracePeriods) {
-      getInstance().gracePeriods = gracePeriods;
-      return this;
-    }
-
-    public Builder addGracePeriod(GracePeriod gracePeriod) {
-      getInstance().gracePeriods = union(getInstance().getGracePeriods(), gracePeriod);
-      return this;
-    }
-
-    public Builder removeGracePeriod(GracePeriod gracePeriod) {
-      getInstance().gracePeriods = CollectionUtils
-          .difference(getInstance().getGracePeriods(), gracePeriod);
-      return this;
-    }
-
-    @Override
-    public Builder setTransferData(DomainTransferData transferData) {
-      getInstance().transferData = transferData;
-      return thisCastToDerived();
-    }
-
-    @Override
-    public Builder setLastTransferTime(DateTime lastTransferTime) {
-      getInstance().lastTransferTime = lastTransferTime;
-      return thisCastToDerived();
-    }
  }
 }
@@ -41,7 +41,6 @@ import google.registry.model.eppinput.ResourceCommand.SingleResourceCommand;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.persistence.VKey;
-import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -447,7 +446,8 @@ public class DomainCommand {
  private static <T extends EppResource> ImmutableMap<String, VKey<T>> loadByForeignKeysCached(
      final Set<String> foreignKeys, final Class<T> clazz, final DateTime now)
      throws InvalidReferencesException {
-    Map<String, ForeignKeyIndex<T>> fkis = ForeignKeyIndex.loadCached(clazz, foreignKeys, now);
+    ImmutableMap<String, ForeignKeyIndex<T>> fkis =
+        ForeignKeyIndex.loadCached(clazz, foreignKeys, now);
    if (!fkis.keySet().equals(foreignKeys)) {
      throw new InvalidReferencesException(
          clazz, ImmutableSet.copyOf(difference(foreignKeys, fkis.keySet())));
@@ -0,0 +1,832 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.domain;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Strings.emptyToNull;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
+import static com.google.common.collect.Sets.difference;
+import static com.google.common.collect.Sets.intersection;
+import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
+import static google.registry.model.EppResourceUtils.setAutomaticTransferSuccessProperties;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.util.CollectionUtils.forceEmptyToNull;
+import static google.registry.util.CollectionUtils.nullToEmpty;
+import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
+import static google.registry.util.CollectionUtils.union;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
+import static google.registry.util.DateTimeUtils.earliestOf;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static google.registry.util.DateTimeUtils.leapSafeAddYears;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.getTldFromDomainName;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.Ignore;
+import com.googlecode.objectify.annotation.IgnoreSave;
+import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.annotation.OnLoad;
+import com.googlecode.objectify.condition.IfNull;
+import google.registry.flows.ResourceFlowUtils;
+import google.registry.model.EppResource;
+import google.registry.model.EppResource.ResourceWithTransferData;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.launch.LaunchNotice;
+import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.eppcommon.StatusValue;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registry.Registry;
+import google.registry.model.transfer.DomainTransferData;
+import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
+import google.registry.util.CollectionUtils;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.Embeddable;
+import javax.persistence.Embedded;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.PostLoad;
+import javax.persistence.Transient;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+
+/**
+ * A persistable domain resource including mutable and non-mutable fields.
+ *
+ * <p>This class deliberately does not include an {@link javax.persistence.Id} so that any
+ * foreign-keyed fields can refer to the proper parent entity's ID, whether we're storing this in
+ * the DB itself or as part of another entity.
+ *
+ * <p>For historical reasons, the name of this class is "DomainContent". Ideally it would be
+ * "DomainBase" for parallelism with the other {@link EppResource} entity classes, but because that
+ * name is already taken by {@link DomainBase} (also for historical reasons), we can't use it. Once
+ * we are no longer on Datastore, we can rename the classes.
+ *
+ * @see <a href="https://tools.ietf.org/html/rfc5731">RFC 5731</a>
+ */
+@MappedSuperclass
+@Embeddable
+@Access(AccessType.FIELD)
+public class DomainContent extends EppResource
+    implements ResourceWithTransferData<DomainTransferData> {
+
+  /** The max number of years that a domain can be registered for, as set by ICANN policy. */
+  public static final int MAX_REGISTRATION_YEARS = 10;
+
+  /** Status values which prohibit DNS information from being published. */
+  private static final ImmutableSet<StatusValue> DNS_PUBLISHING_PROHIBITED_STATUSES =
+      ImmutableSet.of(
+          StatusValue.CLIENT_HOLD,
+          StatusValue.INACTIVE,
+          StatusValue.PENDING_DELETE,
+          StatusValue.SERVER_HOLD);
+
+  /**
+   * Fully qualified domain name (puny-coded), which serves as the foreign key for this domain.
+   *
+   * <p>This is only unique in the sense that for any given lifetime specified as the time range
+   * from (creationTime, deletionTime) there can only be one domain in Datastore with this name.
+   * However, there can be many domains with the same name and non-overlapping lifetimes.
+   *
+   * @invariant fullyQualifiedDomainName == fullyQualifiedDomainName.toLowerCase(Locale.ENGLISH)
+   */
+  // TODO(b/158858642): Rename this to domainName when we are off Datastore
+  @Column(name = "domainName")
+  @Index
+  String fullyQualifiedDomainName;
+
+  /** The top level domain this is under, dernormalized from {@link #fullyQualifiedDomainName}. */
+  @Index String tld;
+
+  /** References to hosts that are the nameservers for the domain. */
+  @Index @Transient Set<VKey<HostResource>> nsHosts;
+
+  /**
+   * The union of the contacts visible via {@link #getContacts} and {@link #getRegistrant}.
+   *
+   * <p>These are stored in one field so that we can query across all contacts at once.
+   */
+  @Transient Set<DesignatedContact> allContacts;
+
+  /**
+   * Contacts as they are stored in cloud SQL.
+   *
+   * <p>This information is duplicated in allContacts, and must be kept in sync with it.
+   */
+  @Ignore VKey<ContactResource> adminContact;
+
+  @Ignore VKey<ContactResource> billingContact;
+  @Ignore VKey<ContactResource> techContact;
+  @Ignore VKey<ContactResource> registrantContact;
+
+  /** Authorization info (aka transfer secret) of the domain. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
+    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
+  })
+  DomainAuthInfo authInfo;
+
+  /**
+   * Data used to construct DS records for this domain.
+   *
+   * <p>This is {@literal @}XmlTransient because it needs to be returned under the "extension" tag
+   * of an info response rather than inside the "infData" tag.
+   */
+  @Transient Set<DelegationSignerData> dsData;
+
+  /**
+   * The claims notice supplied when this application or domain was created, if there was one. It's
+   * {@literal @}XmlTransient because it's not returned in an info response.
+   */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "noticeId.tcnId", column = @Column(name = "launch_notice_tcn_id")),
+    @AttributeOverride(
+        name = "noticeId.validatorId",
+        column = @Column(name = "launch_notice_validator_id")),
+    @AttributeOverride(
+        name = "expirationTime",
+        column = @Column(name = "launch_notice_expiration_time")),
+    @AttributeOverride(
+        name = "acceptedTime",
+        column = @Column(name = "launch_notice_accepted_time")),
+  })
+  LaunchNotice launchNotice;
+
+  /**
+   * Name of first IDN table associated with TLD that matched the characters in this domain label.
+   *
+   * @see google.registry.tldconfig.idn.IdnLabelValidator#findValidIdnTableForTld
+   */
+  @IgnoreSave(IfNull.class)
+  String idnTableName;
+
+  /** Fully qualified host names of this domain's active subordinate hosts. */
+  Set<String> subordinateHosts;
+
+  /** When this domain's registration will expire. */
+  DateTime registrationExpirationTime;
+
+  /**
+   * The poll message associated with this domain being deleted.
+   *
+   * <p>This field should be null if the domain is not in pending delete. If it is, the field should
+   * refer to a {@link PollMessage} timed to when the domain is fully deleted. If the domain is
+   * restored, the message should be deleted.
+   */
+  @Column(name = "deletion_poll_message_id")
+  VKey<PollMessage.OneTime> deletePollMessage;
+
+  /**
+   * The recurring billing event associated with this domain's autorenewals.
+   *
+   * <p>The recurrence should be open ended unless the domain is in pending delete or fully deleted,
+   * in which case it should be closed at the time the delete was requested. Whenever the domain's
+   * {@link #registrationExpirationTime} is changed the recurrence should be closed, a new one
+   * should be created, and this field should be updated to point to the new one.
+   */
+  @Column(name = "billing_recurrence_id")
+  VKey<BillingEvent.Recurring> autorenewBillingEvent;
+
+  /**
+   * The recurring poll message associated with this domain's autorenewals.
+   *
+   * <p>The recurrence should be open ended unless the domain is in pending delete or fully deleted,
+   * in which case it should be closed at the time the delete was requested. Whenever the domain's
+   * {@link #registrationExpirationTime} is changed the recurrence should be closed, a new one
+   * should be created, and this field should be updated to point to the new one.
+   */
+  @Column(name = "autorenew_poll_message_id")
+  VKey<PollMessage.Autorenew> autorenewPollMessage;
+
+  /** The unexpired grace periods for this domain (some of which may not be active yet). */
+  @Transient @ElementCollection Set<GracePeriod> gracePeriods;
+
+  /**
+   * The id of the signed mark that was used to create this domain in sunrise.
+   *
+   * <p>Will only be populated for domains created in sunrise.
+   */
+  @IgnoreSave(IfNull.class)
+  String smdId;
+
+  /** Data about any pending or past transfers on this domain. */
+  DomainTransferData transferData;
+
+  /**
+   * The time that this resource was last transferred.
+   *
+   * <p>Can be null if the resource has never been transferred.
+   */
+  DateTime lastTransferTime;
+
+  @OnLoad
+  void load() {
+    // Reconstitute all of the contacts so that they have VKeys.
+    allContacts =
+        allContacts.stream().map(contact -> contact.reconstitute()).collect(toImmutableSet());
+    setContactFields(allContacts, true);
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Reconstitute the contact list.
+    ImmutableSet.Builder<DesignatedContact> contactsBuilder =
+        new ImmutableSet.Builder<DesignatedContact>();
+
+    if (registrantContact != null) {
+      contactsBuilder.add(
+          DesignatedContact.create(DesignatedContact.Type.REGISTRANT, registrantContact));
+    }
+    if (billingContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.BILLING, billingContact));
+    }
+    if (techContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.TECH, techContact));
+    }
+    if (adminContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.ADMIN, adminContact));
+    }
+
+    allContacts = contactsBuilder.build();
+  }
+
+  public ImmutableSet<String> getSubordinateHosts() {
+    return nullToEmptyImmutableCopy(subordinateHosts);
+  }
+
+  public DateTime getRegistrationExpirationTime() {
+    return registrationExpirationTime;
+  }
+
+  public VKey<PollMessage.OneTime> getDeletePollMessage() {
+    return deletePollMessage;
+  }
+
+  public VKey<BillingEvent.Recurring> getAutorenewBillingEvent() {
+    return autorenewBillingEvent;
+  }
+
+  public VKey<PollMessage.Autorenew> getAutorenewPollMessage() {
+    return autorenewPollMessage;
+  }
+
+  public ImmutableSet<GracePeriod> getGracePeriods() {
+    return nullToEmptyImmutableCopy(gracePeriods);
+  }
+
+  public String getSmdId() {
+    return smdId;
+  }
+
+  @Override
+  public DomainTransferData getTransferData() {
+    return Optional.ofNullable(transferData).orElse(DomainTransferData.EMPTY);
+  }
+
+  @Override
+  public DateTime getLastTransferTime() {
+    return lastTransferTime;
+  }
+
+  @Override
+  public String getForeignKey() {
+    return fullyQualifiedDomainName;
+  }
+
+  public String getDomainName() {
+    return fullyQualifiedDomainName;
+  }
+
+  public ImmutableSet<DelegationSignerData> getDsData() {
+    return nullToEmptyImmutableCopy(dsData);
+  }
+
+  public LaunchNotice getLaunchNotice() {
+    return launchNotice;
+  }
+
+  public String getIdnTableName() {
+    return idnTableName;
+  }
+
+  public ImmutableSet<VKey<HostResource>> getNameservers() {
+    return nullToEmptyImmutableCopy(nsHosts);
+  }
+
+  // Hibernate needs this in order to populate nsHosts but no one else should ever use it
+  @SuppressWarnings("UnusedMethod")
+  private void setNsHosts(Set<VKey<HostResource>> nsHosts) {
+    this.nsHosts = nsHosts;
+  }
+
+  public final String getCurrentSponsorClientId() {
+    return getPersistedCurrentSponsorClientId();
+  }
+
+  /** Returns true if DNS information should be published for the given domain. */
+  public boolean shouldPublishToDns() {
+    return intersection(getStatusValues(), DNS_PUBLISHING_PROHIBITED_STATUSES).isEmpty();
+  }
+
+  /**
+   * Returns the Registry Grace Period Statuses for this domain.
+   *
+   * <p>This collects all statuses from the domain's {@link GracePeriod} entries and also adds the
+   * PENDING_DELETE status if needed.
+   */
+  public ImmutableSet<GracePeriodStatus> getGracePeriodStatuses() {
+    Set<GracePeriodStatus> gracePeriodStatuses = new HashSet<>();
+    for (GracePeriod gracePeriod : getGracePeriods()) {
+      gracePeriodStatuses.add(gracePeriod.getType());
+    }
+    if (getStatusValues().contains(StatusValue.PENDING_DELETE)
+        && !gracePeriodStatuses.contains(GracePeriodStatus.REDEMPTION)) {
+      gracePeriodStatuses.add(GracePeriodStatus.PENDING_DELETE);
+    }
+    return ImmutableSet.copyOf(gracePeriodStatuses);
+  }
+
+  /** Returns the subset of grace periods having the specified type. */
+  public ImmutableSet<GracePeriod> getGracePeriodsOfType(GracePeriodStatus gracePeriodType) {
+    ImmutableSet.Builder<GracePeriod> builder = new ImmutableSet.Builder<>();
+    for (GracePeriod gracePeriod : getGracePeriods()) {
+      if (gracePeriod.getType() == gracePeriodType) {
+        builder.add(gracePeriod);
+      }
+    }
+    return builder.build();
+  }
+
+  @Override
+  public DomainContent cloneProjectedAtTime(final DateTime now) {
+    return cloneDomainProjectedAtTime(this, now);
+  }
+
+  /**
+   * The logic in this method, which handles implicit server approval of transfers, very closely
+   * parallels the logic in {@code DomainTransferApproveFlow} which handles explicit client
+   * approvals.
+   */
+  protected static <T extends DomainContent> T cloneDomainProjectedAtTime(T domain, DateTime now) {
+    DomainTransferData transferData = domain.getTransferData();
+    DateTime transferExpirationTime = transferData.getPendingTransferExpirationTime();
+
+    // If there's a pending transfer that has expired, handle it.
+    if (TransferStatus.PENDING.equals(transferData.getTransferStatus())
+        && isBeforeOrAt(transferExpirationTime, now)) {
+      // Project until just before the transfer time. This will handle the case of an autorenew
+      // before the transfer was even requested or during the request period.
+      // If the transfer time is precisely the moment that the domain expires, there will not be an
+      // autorenew billing event (since we end the recurrence at transfer time and recurrences are
+      // exclusive of their ending), and we can just proceed with the transfer.
+      T domainAtTransferTime =
+          cloneDomainProjectedAtTime(domain, transferExpirationTime.minusMillis(1));
+
+      DateTime expirationDate = transferData.getTransferredRegistrationExpirationTime();
+      if (expirationDate == null) {
+        // Extend the registration by the correct number of years from the expiration time
+        // that was current on the domain right before the transfer, capped at 10 years from
+        // the moment of the transfer.
+        expirationDate =
+            ResourceFlowUtils.computeExDateForApprovalTime(
+                domainAtTransferTime, transferExpirationTime, transferData.getTransferPeriod());
+      }
+      // If we are within an autorenew grace period, the transfer will subsume the autorenew. There
+      // will already be a cancellation written in advance by the transfer request flow, so we don't
+      // need to worry about billing, but we do need to cancel out the expiration time increase.
+      // The transfer period saved in the transfer data will be one year, unless the superuser
+      // extension set the transfer period to zero.
+      // Set the expiration, autorenew events, and grace period for the transfer. (Transfer ends
+      // all other graces).
+      Builder builder =
+          domainAtTransferTime
+              .asBuilder()
+              .setRegistrationExpirationTime(expirationDate)
+              // Set the speculatively-written new autorenew events as the domain's autorenew
+              // events.
+              .setAutorenewBillingEvent(transferData.getServerApproveAutorenewEvent())
+              .setAutorenewPollMessage(transferData.getServerApproveAutorenewPollMessage());
+      if (transferData.getTransferPeriod().getValue() == 1) {
+        // Set the grace period using a key to the prescheduled transfer billing event.  Not using
+        // GracePeriod.forBillingEvent() here in order to avoid the actual Datastore fetch.
+        builder.setGracePeriods(
+            ImmutableSet.of(
+                GracePeriod.create(
+                    GracePeriodStatus.TRANSFER,
+                    transferExpirationTime.plus(
+                        Registry.get(domain.getTld()).getTransferGracePeriodLength()),
+                    transferData.getGainingClientId(),
+                    transferData.getServerApproveBillingEvent())));
+      } else {
+        // There won't be a billing event, so we don't need a grace period
+        builder.setGracePeriods(ImmutableSet.of());
+      }
+      // Set all remaining transfer properties.
+      setAutomaticTransferSuccessProperties(builder, transferData);
+      builder
+          .setLastEppUpdateTime(transferExpirationTime)
+          .setLastEppUpdateClientId(transferData.getGainingClientId());
+      // Finish projecting to now.
+      return (T) builder.build().cloneProjectedAtTime(now);
+    }
+
+    Optional<DateTime> newLastEppUpdateTime = Optional.empty();
+
+    // There is no transfer. Do any necessary autorenews for active domains.
+
+    Builder builder = domain.asBuilder();
+    if (isBeforeOrAt(domain.getRegistrationExpirationTime(), now)
+        && END_OF_TIME.equals(domain.getDeletionTime())) {
+      // Autorenew by the number of years between the old expiration time and now.
+      DateTime lastAutorenewTime =
+          leapSafeAddYears(
+              domain.getRegistrationExpirationTime(),
+              new Interval(domain.getRegistrationExpirationTime(), now).toPeriod().getYears());
+      DateTime newExpirationTime = lastAutorenewTime.plusYears(1);
+      builder
+          .setRegistrationExpirationTime(newExpirationTime)
+          .addGracePeriod(
+              GracePeriod.createForRecurring(
+                  GracePeriodStatus.AUTO_RENEW,
+                  lastAutorenewTime.plus(
+                      Registry.get(domain.getTld()).getAutoRenewGracePeriodLength()),
+                  domain.getCurrentSponsorClientId(),
+                  domain.getAutorenewBillingEvent()));
+      newLastEppUpdateTime = Optional.of(lastAutorenewTime);
+    }
+
+    // Remove any grace periods that have expired.
+    T almostBuilt = (T) builder.build();
+    builder = almostBuilt.asBuilder();
+    for (GracePeriod gracePeriod : almostBuilt.getGracePeriods()) {
+      if (isBeforeOrAt(gracePeriod.getExpirationTime(), now)) {
+        builder.removeGracePeriod(gracePeriod);
+        if (!newLastEppUpdateTime.isPresent()
+            || isBeforeOrAt(newLastEppUpdateTime.get(), gracePeriod.getExpirationTime())) {
+          newLastEppUpdateTime = Optional.of(gracePeriod.getExpirationTime());
+        }
+      }
+    }
+
+    // It is possible that the lastEppUpdateClientId is different from current sponsor client
+    // id, so we have to do the comparison instead of having one variable just storing the most
+    // recent time.
+    if (newLastEppUpdateTime.isPresent()) {
+      if (domain.getLastEppUpdateTime() == null
+          || newLastEppUpdateTime.get().isAfter(domain.getLastEppUpdateTime())) {
+        builder
+            .setLastEppUpdateTime(newLastEppUpdateTime.get())
+            .setLastEppUpdateClientId(domain.getCurrentSponsorClientId());
+      }
+    }
+
+    // Handle common properties like setting or unsetting linked status. This also handles the
+    // general case of pending transfers for other resource types, but since we've always handled
+    // a pending transfer by this point that's a no-op for domains.
+    projectResourceOntoBuilderAtTime(almostBuilt, builder, now);
+    return (T) builder.build();
+  }
+
+  /** Return what the expiration time would be if the given number of years were added to it. */
+  public static DateTime extendRegistrationWithCap(
+      DateTime now, DateTime currentExpirationTime, @Nullable Integer extendedRegistrationYears) {
+    // We must cap registration at the max years (aka 10), even if that truncates the last year.
+    return earliestOf(
+        leapSafeAddYears(
+            currentExpirationTime, Optional.ofNullable(extendedRegistrationYears).orElse(0)),
+        leapSafeAddYears(now, MAX_REGISTRATION_YEARS));
+  }
+
+  /** Loads and returns the fully qualified host names of all linked nameservers. */
+  public ImmutableSortedSet<String> loadNameserverHostNames() {
+    return ofy().load()
+        .keys(getNameservers().stream().map(VKey::getOfyKey).collect(toImmutableSet())).values()
+        .stream()
+        .map(HostResource::getHostName)
+        .collect(toImmutableSortedSet(Ordering.natural()));
+  }
+
+  /** A key to the registrant who registered this domain. */
+  public VKey<ContactResource> getRegistrant() {
+    return registrantContact;
+  }
+
+  public VKey<ContactResource> getAdminContact() {
+    return adminContact;
+  }
+
+  public VKey<ContactResource> getBillingContact() {
+    return billingContact;
+  }
+
+  public VKey<ContactResource> getTechContact() {
+    return techContact;
+  }
+
+  /** Associated contacts for the domain (other than registrant). */
+  public ImmutableSet<DesignatedContact> getContacts() {
+    return nullToEmpty(allContacts).stream()
+        .filter(IS_REGISTRANT.negate())
+        .collect(toImmutableSet());
+  }
+
+  public DomainAuthInfo getAuthInfo() {
+    return authInfo;
+  }
+
+  /** Returns all referenced contacts from this domain or application. */
+  public ImmutableSet<VKey<ContactResource>> getReferencedContacts() {
+    return nullToEmptyImmutableCopy(allContacts).stream()
+        .map(DesignatedContact::getContactKey)
+        .filter(Objects::nonNull)
+        .collect(toImmutableSet());
+  }
+
+  public String getTld() {
+    return tld;
+  }
+
+  /**
+   * Sets the individual contact fields from {@code contacts}.
+   *
+   * <p>The registrant field is only set if {@code includeRegistrant} is true, as this field needs
+   * to be set in some circumstances but not in others.
+   */
+  protected void setContactFields(Set<DesignatedContact> contacts, boolean includeRegistrant) {
+    // Set the individual contact fields.
+    for (DesignatedContact contact : contacts) {
+      switch (contact.getType()) {
+        case BILLING:
+          billingContact = contact.getContactKey();
+          break;
+        case TECH:
+          techContact = contact.getContactKey();
+          break;
+        case ADMIN:
+          adminContact = contact.getContactKey();
+          break;
+        case REGISTRANT:
+          if (includeRegistrant) {
+            registrantContact = contact.getContactKey();
+          }
+          break;
+        default:
+          throw new IllegalArgumentException("Unknown contact resource type: " + contact.getType());
+      }
+    }
+  }
+
+  @Override
+  public VKey<DomainBase> createVKey() {
+    return VKey.create(DomainBase.class, getRepoId(), Key.create(this));
+  }
+
+  public static VKey<DomainBase> createVKey(Key key) {
+    return VKey.create(DomainBase.class, key.getName(), key);
+  }
+
+  /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
+  protected static final Predicate<DesignatedContact> IS_REGISTRANT =
+      (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);
+
+  /** An override of {@link EppResource#asBuilder} with tighter typing. */
+  @Override
+  public Builder asBuilder() {
+    return new Builder<>(clone(this));
+  }
+
+  /** A builder for constructing {@link DomainBase}, since it is immutable. */
+  public static class Builder<T extends DomainContent, B extends Builder<T, B>>
+      extends EppResource.Builder<T, B> implements BuilderWithTransferData<DomainTransferData, B> {
+
+    public Builder() {}
+
+    Builder(T instance) {
+      super(instance);
+    }
+
+    @Override
+    public T build() {
+      T instance = getInstance();
+      // If TransferData is totally empty, set it to null.
+      if (DomainTransferData.EMPTY.equals(getInstance().transferData)) {
+        setTransferData(null);
+      }
+      // A DomainBase has status INACTIVE if there are no nameservers.
+      if (getInstance().getNameservers().isEmpty()) {
+        addStatusValue(StatusValue.INACTIVE);
+      } else { // There are nameservers, so make sure INACTIVE isn't there.
+        removeStatusValue(StatusValue.INACTIVE);
+      }
+
+      checkArgumentNotNull(emptyToNull(instance.fullyQualifiedDomainName), "Missing domainName");
+      if (instance.getRegistrant() == null
+          && instance.allContacts.stream().anyMatch(IS_REGISTRANT)) {
+        throw new IllegalArgumentException("registrant is null but is in allContacts");
+      }
+      checkArgumentNotNull(instance.getRegistrant(), "Missing registrant");
+      instance.tld = getTldFromDomainName(instance.fullyQualifiedDomainName);
+      return super.build();
+    }
+
+    public B setDomainName(String domainName) {
+      checkArgument(
+          domainName.equals(canonicalizeDomainName(domainName)),
+          "Domain name must be in puny-coded, lower-case form");
+      getInstance().fullyQualifiedDomainName = domainName;
+      return thisCastToDerived();
+    }
+
+    public B setDsData(ImmutableSet<DelegationSignerData> dsData) {
+      getInstance().dsData = dsData;
+      return thisCastToDerived();
+    }
+
+    public B setRegistrant(VKey<ContactResource> registrant) {
+      // Replace the registrant contact inside allContacts.
+      getInstance().allContacts =
+          union(
+              getInstance().getContacts(),
+              DesignatedContact.create(
+                  DesignatedContact.Type.REGISTRANT, checkArgumentNotNull(registrant)));
+
+      // Set the registrant field specifically.
+      getInstance().registrantContact = registrant;
+      return thisCastToDerived();
+    }
+
+    public B setAuthInfo(DomainAuthInfo authInfo) {
+      getInstance().authInfo = authInfo;
+      return thisCastToDerived();
+    }
+
+    public B setNameservers(VKey<HostResource> nameserver) {
+      getInstance().nsHosts = ImmutableSet.of(nameserver);
+      return thisCastToDerived();
+    }
+
+    public B setNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
+      getInstance().nsHosts = forceEmptyToNull(nameservers);
+      return thisCastToDerived();
+    }
+
+    public B addNameserver(VKey<HostResource> nameserver) {
+      return addNameservers(ImmutableSet.of(nameserver));
+    }
+
+    public B addNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
+      return setNameservers(
+          ImmutableSet.copyOf(Sets.union(getInstance().getNameservers(), nameservers)));
+    }
+
+    public B removeNameserver(VKey<HostResource> nameserver) {
+      return removeNameservers(ImmutableSet.of(nameserver));
+    }
+
+    public B removeNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
+      return setNameservers(
+          ImmutableSet.copyOf(difference(getInstance().getNameservers(), nameservers)));
+    }
+
+    public B setContacts(DesignatedContact contact) {
+      return setContacts(ImmutableSet.of(contact));
+    }
+
+    public B setContacts(ImmutableSet<DesignatedContact> contacts) {
+      checkArgument(contacts.stream().noneMatch(IS_REGISTRANT), "Registrant cannot be a contact");
+
+      // Replace the non-registrant contacts inside allContacts.
+      getInstance().allContacts =
+          Streams.concat(
+                  nullToEmpty(getInstance().allContacts).stream().filter(IS_REGISTRANT),
+                  contacts.stream())
+              .collect(toImmutableSet());
+
+      // Set the individual fields.
+      getInstance().setContactFields(contacts, false);
+      return thisCastToDerived();
+    }
+
+    public B addContacts(ImmutableSet<DesignatedContact> contacts) {
+      return setContacts(ImmutableSet.copyOf(Sets.union(getInstance().getContacts(), contacts)));
+    }
+
+    public B removeContacts(ImmutableSet<DesignatedContact> contacts) {
+      return setContacts(ImmutableSet.copyOf(difference(getInstance().getContacts(), contacts)));
+    }
+
+    public B setLaunchNotice(LaunchNotice launchNotice) {
+      getInstance().launchNotice = launchNotice;
+      return thisCastToDerived();
+    }
+
+    public B setIdnTableName(String idnTableName) {
+      getInstance().idnTableName = idnTableName;
+      return thisCastToDerived();
+    }
+
+    public B setSubordinateHosts(ImmutableSet<String> subordinateHosts) {
+      getInstance().subordinateHosts = subordinateHosts;
+      return thisCastToDerived();
+    }
+
+    public B addSubordinateHost(String hostToAdd) {
+      return setSubordinateHosts(
+          ImmutableSet.copyOf(union(getInstance().getSubordinateHosts(), hostToAdd)));
+    }
+
+    public B removeSubordinateHost(String hostToRemove) {
+      return setSubordinateHosts(
+          ImmutableSet.copyOf(
+              CollectionUtils.difference(getInstance().getSubordinateHosts(), hostToRemove)));
+    }
+
+    public B setRegistrationExpirationTime(DateTime registrationExpirationTime) {
+      getInstance().registrationExpirationTime = registrationExpirationTime;
+      return thisCastToDerived();
+    }
+
+    public B setDeletePollMessage(VKey<PollMessage.OneTime> deletePollMessage) {
+      getInstance().deletePollMessage = deletePollMessage;
+      return thisCastToDerived();
+    }
+
+    public B setAutorenewBillingEvent(VKey<BillingEvent.Recurring> autorenewBillingEvent) {
+      getInstance().autorenewBillingEvent = autorenewBillingEvent;
+      return thisCastToDerived();
+    }
+
+    public B setAutorenewPollMessage(VKey<PollMessage.Autorenew> autorenewPollMessage) {
+      getInstance().autorenewPollMessage = autorenewPollMessage;
+      return thisCastToDerived();
+    }
+
+    public B setSmdId(String smdId) {
+      getInstance().smdId = smdId;
+      return thisCastToDerived();
+    }
+
+    public B setGracePeriods(ImmutableSet<GracePeriod> gracePeriods) {
+      getInstance().gracePeriods = gracePeriods;
+      return thisCastToDerived();
+    }
+
+    public B addGracePeriod(GracePeriod gracePeriod) {
+      getInstance().gracePeriods = union(getInstance().getGracePeriods(), gracePeriod);
+      return thisCastToDerived();
+    }
+
+    public B removeGracePeriod(GracePeriod gracePeriod) {
+      getInstance().gracePeriods =
+          CollectionUtils.difference(getInstance().getGracePeriods(), gracePeriod);
+      return thisCastToDerived();
+    }
+
+    @Override
+    public B setTransferData(DomainTransferData transferData) {
+      getInstance().transferData = transferData;
+      return thisCastToDerived();
+    }
+
+    @Override
+    public B setLastTransferTime(DateTime lastTransferTime) {
+      getInstance().lastTransferTime = lastTransferTime;
+      return thisCastToDerived();
+    }
+  }
+}
@@ -0,0 +1,111 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.domain;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.EppResource;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.host.HostResource;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import java.util.Set;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.Entity;
+import javax.persistence.JoinTable;
+
+/**
+ * A persisted history entry representing an EPP modification to a domain.
+ *
+ * <p>In addition to the general history fields (e.g. action time, registrar ID) we also persist a
+ * copy of the domain entity at this point in time. We persist a raw {@link DomainContent} so that
+ * the foreign-keyed fields in that class can refer to this object.
+ */
+@Entity
+@javax.persistence.Table(
+    indexes = {
+        @javax.persistence.Index(columnList = "creationTime"),
+        @javax.persistence.Index(columnList = "historyRegistrarId"),
+        @javax.persistence.Index(columnList = "historyType"),
+        @javax.persistence.Index(columnList = "historyModificationTime")
+    })
+public class DomainHistory extends HistoryEntry {
+  // Store DomainContent instead of DomainBase so we don't pick up its @Id
+  DomainContent domainContent;
+
+  @Column(nullable = false)
+  VKey<DomainBase> domainRepoId;
+
+  @ElementCollection
+  @JoinTable(name = "DomainHistoryHost")
+  @Access(AccessType.PROPERTY)
+  @Column(name = "host_repo_id")
+  public Set<VKey<HostResource>> getNsHosts() {
+    return domainContent.nsHosts;
+  }
+
+  /** The state of the {@link DomainContent} object at this point in time. */
+  public DomainContent getDomainContent() {
+    return domainContent;
+  }
+
+  /** The key to the {@link ContactResource} this is based off of. */
+  public VKey<DomainBase> getDomainRepoId() {
+    return domainRepoId;
+  }
+
+  // Hibernate needs this in order to populate nsHosts but no one else should ever use it
+  @SuppressWarnings("UnusedMethod")
+  private void setNsHosts(Set<VKey<HostResource>> nsHosts) {
+    if (domainContent != null) {
+      domainContent.nsHosts = nsHosts;
+    }
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder(clone(this));
+  }
+
+  public static class Builder extends HistoryEntry.Builder<DomainHistory, DomainHistory.Builder> {
+
+    public Builder() {}
+
+    public Builder(DomainHistory instance) {
+      super(instance);
+    }
+
+    public Builder setDomainContent(DomainContent domainContent) {
+      getInstance().domainContent = domainContent;
+      return this;
+    }
+
+    public Builder setDomainRepoId(VKey<DomainBase> domainRepoId) {
+      getInstance().domainRepoId = domainRepoId;
+      domainRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
+      return this;
+    }
+
+    // We can remove this once all HistoryEntries are converted to History objects
+    @Override
+    public Builder setParent(Key<? extends EppResource> parent) {
+      super.setParent(parent);
+      getInstance().domainRepoId = VKey.create(DomainBase.class, parent.getName(), parent);
+      return this;
+    }
+  }
+}
@@ -17,12 +17,13 @@ package google.registry.model.domain;
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.persistence.VKey;
 import javax.annotation.Nullable;
 import javax.persistence.Column;
 import javax.persistence.GeneratedValue;
@@ -57,18 +58,18 @@ public class GracePeriod extends ImmutableObject {

  /**
   * The one-time billing event corresponding to the action that triggered this grace period, or
-   * null if not applicable.  Not set for autorenew grace periods (which instead use the field
-   * {@code billingEventRecurring}) or for redemption grace periods (since deletes have no cost).
+   * null if not applicable. Not set for autorenew grace periods (which instead use the field {@code
+   * billingEventRecurring}) or for redemption grace periods (since deletes have no cost).
   */
  // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  Key<BillingEvent.OneTime> billingEventOneTime = null;
+  VKey<BillingEvent.OneTime> billingEventOneTime = null;

  /**
   * The recurring billing event corresponding to the action that triggered this grace period, if
   * applicable - i.e. if the action was an autorenew - or null in all other cases.
   */
  // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  Key<BillingEvent.Recurring> billingEventRecurring = null;
+  VKey<BillingEvent.Recurring> billingEventRecurring = null;

  public GracePeriodStatus getType() {
    return type;
@@ -91,8 +92,7 @@ public class GracePeriod extends ImmutableObject {
   * Returns the one time billing event. The value will only be non-null if the type of this grace
   * period is not AUTO_RENEW.
   */
-
-  public Key<BillingEvent.OneTime> getOneTimeBillingEvent() {
+  public VKey<BillingEvent.OneTime> getOneTimeBillingEvent() {
    return billingEventOneTime;
  }

@@ -100,16 +100,16 @@ public class GracePeriod extends ImmutableObject {
   * Returns the recurring billing event. The value will only be non-null if the type of this grace
   * period is AUTO_RENEW.
   */
-  public Key<BillingEvent.Recurring> getRecurringBillingEvent() {
+  public VKey<BillingEvent.Recurring> getRecurringBillingEvent() {
    return billingEventRecurring;
  }

  private static GracePeriod createInternal(
-       GracePeriodStatus type,
-       DateTime expirationTime,
-       String clientId,
-       @Nullable Key<BillingEvent.OneTime> billingEventOneTime,
-       @Nullable Key<BillingEvent.Recurring> billingEventRecurring) {
+      GracePeriodStatus type,
+      DateTime expirationTime,
+      String clientId,
+      @Nullable VKey<BillingEvent.OneTime> billingEventOneTime,
+      @Nullable VKey<BillingEvent.Recurring> billingEventRecurring) {
    checkArgument((billingEventOneTime == null) || (billingEventRecurring == null),
        "A grace period can have at most one billing event");
    checkArgument(
@@ -127,15 +127,15 @@ public class GracePeriod extends ImmutableObject {
  /**
   * Creates a GracePeriod for an (optional) OneTime billing event.
   *
-   * <p>Normal callers should always use {@link #forBillingEvent} instead, assuming they do not
-   * need to avoid loading the BillingEvent from Datastore.  This method should typically be
-   * called only from test code to explicitly construct GracePeriods.
+   * <p>Normal callers should always use {@link #forBillingEvent} instead, assuming they do not need
+   * to avoid loading the BillingEvent from Datastore. This method should typically be called only
+   * from test code to explicitly construct GracePeriods.
   */
  public static GracePeriod create(
      GracePeriodStatus type,
      DateTime expirationTime,
      String clientId,
-      @Nullable Key<BillingEvent.OneTime> billingEventOneTime) {
+      @Nullable VKey<BillingEvent.OneTime> billingEventOneTime) {
    return createInternal(type, expirationTime, clientId, billingEventOneTime, null);
  }

@@ -144,7 +144,7 @@ public class GracePeriod extends ImmutableObject {
      GracePeriodStatus type,
      DateTime expirationTime,
      String clientId,
-      Key<BillingEvent.Recurring> billingEventRecurring) {
+      VKey<Recurring> billingEventRecurring) {
    checkArgumentNotNull(billingEventRecurring, "billingEventRecurring cannot be null");
    return createInternal(type, expirationTime, clientId, null, billingEventRecurring);
  }
@@ -159,6 +159,6 @@ public class GracePeriod extends ImmutableObject {
  public static GracePeriod forBillingEvent(
      GracePeriodStatus type, BillingEvent.OneTime billingEvent) {
    return create(
-        type, billingEvent.getBillingTime(), billingEvent.getClientId(), Key.create(billingEvent));
+        type, billingEvent.getBillingTime(), billingEvent.getClientId(), billingEvent.createVKey());
  }
 }
@@ -15,7 +15,6 @@
 package google.registry.model.domain.fee;

 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.eppinput.EppInput.CommandExtension;
 import org.joda.money.CurrencyUnit;

@@ -40,7 +39,7 @@ public interface FeeCheckCommandExtension<
   */
  CurrencyUnit getCurrency();

-  ImmutableSet<C> getItems();
+  ImmutableList<C> getItems();

  R createResponse(ImmutableList<? extends FeeCheckResponseExtensionItem> items);
 }
--- a/Show More
+++ b/Show More