Use Java 11 in GCB to build release candidates (#736)

* Add use of interval data type

* Add support for Millis

* Use Java-object type

* Change column type for relock_duration

* add years and months

* Add tests for hours, minutes, and seconds

* Add javadoc describing how joda duration is stored

* Add test for lots of days

.gitignore

@@ -79,6 +79,10 @@ nomulus.iml
 nomulus.ipr
 nomulus.iws
 
+# Auto-generated java classes by Intellij
+*/src/main/generated/
+*/src/test/generated_tests/
+
 # VScode
 .vscode
 

build.gradle

@@ -25,7 +25,7 @@ buildscript {
 
   dependencies {
     classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.0.1'
-    classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.6.1"
+    classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.6.1'
     classpath 'org.sonatype.aether:aether-api:1.13.1'
     classpath 'org.sonatype.aether:aether-impl:1.13.1'
   }
@@ -35,9 +35,6 @@ plugins {
   // Java static analysis plugins. Keep versions consistent with
   // ./buildSrc/build.gradle
   id 'nebula.lint' version '16.0.2'
-  // TODO(weiminyu): consider remove net.ltgt.apt. Gradle 5.2+
-  //  has similar functionalities.
-  id 'net.ltgt.apt' version '0.19' apply false
   id 'net.ltgt.errorprone' version '0.6.1'
   id 'checkstyle'
   id 'com.github.johnrengelman.shadow' version '5.1.0'
@@ -49,6 +46,7 @@ plugins {
   id 'com.diffplug.gradle.spotless' version '3.25.0'
 
   id 'jacoco'
+  id 'com.dorongold.task-tree' version '1.5'
 }
 
 wrapper {
@@ -289,12 +287,6 @@ subprojects {
 
   project.tasks.test.dependsOn runPresubmits
 
-// Path to code generated with annotation processors. Note that this path is
-// chosen by the 'net.ltgt.apt' plugin, and may change if IDE-specific plugins
-// are applied, e.g., 'idea' or 'eclipse'
-  def aptGeneratedDir = "${project.buildDir}/generated/source/apt/main"
-  def aptGeneratedTestDir = "${project.buildDir}/generated/source/apt/test"
-
   def commonlyExcludedResources = ['**/*.java', '**/BUILD']
 
   project.ext.javaDir = "${project.projectDir}/src/main/java"
@@ -303,18 +295,12 @@ subprojects {
 
   sourceSets {
     main {
-      java {
-        srcDirs += aptGeneratedDir
-      }
       resources {
         srcDirs += project.ext.javaDir
         exclude commonlyExcludedResources
       }
     }
     test {
-      java {
-        srcDirs += aptGeneratedTestDir
-      }
       resources {
         srcDirs += project.ext.javaTestDir
         exclude commonlyExcludedResources
@@ -322,9 +308,14 @@ subprojects {
     }
   }
 
-  javadocSource << project.sourceSets.main.allJava
-  javadocClasspath << project.sourceSets.main.compileClasspath
-  javadocDependentTasks << project.tasks.compileJava
+  // No need to produce javadoc for the docs subproject, which has no APIs to
+  // expose to users.
+  if (project.name != 'docs') {
+    javadocSource << project.sourceSets.main.allJava
+    javadocClasspath << project.sourceSets.main.compileClasspath
+    javadocClasspath << "${buildDir}/generated/sources/annotationProcessor/java/main"
+    javadocDependentTasks << project.tasks.compileJava
+  }
 }
 
 // If "-P verboseTestOutput=true" is passed in, configure all subprojects to dump all of their
@@ -443,6 +434,7 @@ task javaIncrementalFormatDryRun {
     println("${invokeJavaDiffFormatScript("show")}")
   }
 }
+tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)
 
 // Checks if modified lines in Java source files need reformatting.
 // Note that this task processes modified Java files in the entire repository.
@@ -452,8 +444,6 @@ task javaIncrementalFormatApply {
   }
 }
 
-tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)
-
 task javadoc(type: Javadoc) {
   source javadocSource
   classpath = files(javadocClasspath)
@@ -468,4 +458,16 @@ task javadoc(type: Javadoc) {
 
 tasks.build.dependsOn(tasks.javadoc)
 
+// Task for doing development on core Nomulus.
+// This fixes code formatting automatically as necessary, builds and tests the
+// core Nomulus codebase, and runs all presubmits.
+task coreDev {
+  dependsOn 'javaIncrementalFormatApply'
+  dependsOn 'javadoc'
+  dependsOn 'checkDependenciesDotGradle'
+  dependsOn 'checkLicense'
+  dependsOn ':core:check'
+  dependsOn 'assemble'
+}
+
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }

buildSrc/build.gradle

@@ -26,9 +26,6 @@ buildscript {
 plugins {
   // Java static analysis plugins. Keep versions consistent with ../build.gradle
   id 'nebula.lint' version '16.0.2'
-  // Config helper for annotation processors such as AutoValue and Dagger.
-  // Ensures that source code is generated at an appropriate location.
-  id 'net.ltgt.apt' version '0.19' apply false
   id 'net.ltgt.errorprone' version '0.6.1'
   id 'checkstyle'
   id 'com.diffplug.gradle.spotless' version '3.25.0'
@@ -83,10 +80,8 @@ dependencies {
   annotationProcessor deps['com.google.auto.value:auto-value']
   testCompile deps['com.google.truth:truth']
   testCompile deps['com.google.truth.extensions:truth-java8-extension']
-  testCompile deps['junit:junit']
   testCompile deps['org.junit.jupiter:junit-jupiter-api']
   testCompile deps['org.junit.jupiter:junit-jupiter-engine']
-  testCompile deps['org.junit.vintage:junit-vintage-engine']
   testCompile deps['org.mockito:mockito-core']
 }
 

buildSrc/gradle/dependency-locks/buildscript-classpath.lockfile

@@ -19,9 +19,7 @@ commons-lang:commons-lang:2.6
 javax.inject:javax.inject:1
 junit:junit:4.12
 nebula.lint:nebula.lint.gradle.plugin:16.0.2
-net.ltgt.apt:net.ltgt.apt.gradle.plugin:0.19
 net.ltgt.errorprone:net.ltgt.errorprone.gradle.plugin:0.6.1
-net.ltgt.gradle:gradle-apt-plugin:0.19
 net.ltgt.gradle:gradle-errorprone-plugin:0.6.1
 org.apache.commons:commons-lang3:3.8.1
 org.apache.maven:maven-artifact:3.6.2

buildSrc/gradle/dependency-locks/testCompile.lockfile

@@ -49,7 +49,7 @@ javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
@@ -61,12 +61,11 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -49,7 +49,7 @@ javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
@@ -61,12 +61,11 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testRuntime.lockfile

@@ -49,7 +49,7 @@ javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
@@ -61,12 +61,11 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -49,7 +49,7 @@ javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
@@ -61,12 +61,11 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java

@@ -57,6 +57,10 @@ final class GcsPluginUtils {
     return file.toPath().toAbsolutePath().normalize();
   }
 
+  static Path toNormalizedPath(Path file) {
+    return file.toAbsolutePath().normalize();
+  }
+
   static String getContentType(String fileName) {
     return EXTENSION_TO_CONTENT_TYPE.getOrDefault(
         Files.getFileExtension(fileName), DEFAULT_CONTENT_TYPE);

buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java

@@ -26,13 +26,10 @@ import com.google.common.collect.ImmutableSet;
 import google.registry.gradle.plugin.ProjectData.TaskData;
 import java.io.File;
 import java.nio.file.Paths;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import org.junit.jupiter.api.Test;
 
 /** Tests for {@link CoverPageGenerator} */
-@RunWith(JUnit4.class)
-public final class CoverPageGeneratorTest {
+final class CoverPageGeneratorTest {
 
   private static final ProjectData EMPTY_PROJECT =
       ProjectData.builder()
@@ -88,14 +85,14 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testGetFilesToUpload_entryPoint_isIndexHtml() {
+  void testGetFilesToUpload_entryPoint_isIndexHtml() {
     CoverPageGenerator coverPageGenerator = new CoverPageGenerator(EMPTY_PROJECT);
     assertThat(coverPageGenerator.getFilesToUpload().entryPoint())
         .isEqualTo(Paths.get("index.html"));
   }
 
   @Test
-  public void testGetFilesToUpload_containsEntryFile() {
+  void testGetFilesToUpload_containsEntryFile() {
     String content = getContentOfGeneratedFile(EMPTY_PROJECT, "index.html");
     assertThat(content)
         .contains(
@@ -103,7 +100,7 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCoverPage_showsFailedTask() {
+  void testCoverPage_showsFailedTask() {
     String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_FAILURE).build());
     assertThat(content).contains("task-failure");
     assertThat(content).contains("<p>FAILURE</p>");
@@ -112,7 +109,7 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCoverPage_showsSuccessfulTask() {
+  void testCoverPage_showsSuccessfulTask() {
     String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_SUCCESS).build());
     assertThat(content).contains("task-success");
     assertThat(content).doesNotContain("<p>FAILURE</p>");
@@ -121,7 +118,7 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCoverPage_showsUpToDateTask() {
+  void testCoverPage_showsUpToDateTask() {
     String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_UP_TO_DATE).build());
     assertThat(content).contains("task-up-to-date");
     assertThat(content).doesNotContain("<p>FAILURE</p>");
@@ -130,11 +127,10 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCoverPage_failedAreFirst() {
+  void testCoverPage_failedAreFirst() {
     String content =
         getCoverPage(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                 .addTask(EMPTY_TASK_UP_TO_DATE)
                 .addTask(EMPTY_TASK_FAILURE)
                 .addTask(EMPTY_TASK_SUCCESS)
@@ -149,11 +145,10 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCoverPage_failingTask_statusIsFailure() {
+  void testCoverPage_failingTask_statusIsFailure() {
     String content =
         getCoverPage(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                 .addTask(EMPTY_TASK_UP_TO_DATE)
                 .addTask(EMPTY_TASK_FAILURE)
                 .addTask(EMPTY_TASK_SUCCESS)
@@ -162,11 +157,10 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCoverPage_noFailingTask_statusIsSuccess() {
+  void testCoverPage_noFailingTask_statusIsSuccess() {
     String content =
         getCoverPage(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                 .addTask(EMPTY_TASK_UP_TO_DATE)
                 .addTask(EMPTY_TASK_SUCCESS)
                 .build());
@@ -174,7 +168,7 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testGetFilesToUpload_containsCssFile() {
+  void testGetFilesToUpload_containsCssFile() {
     ImmutableMap<String, String> files = getGeneratedFiles(EMPTY_PROJECT);
     assertThat(files).containsKey(filenameJoiner.join("css", "style.css"));
     assertThat(files.get(filenameJoiner.join("css", "style.css"))).contains("body {");
@@ -183,14 +177,12 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCreateReportFiles_taskWithLog() {
+  void testCreateReportFiles_taskWithLog() {
     ImmutableMap<String, String> files =
         getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                 .addTask(
-                    EMPTY_TASK_SUCCESS
-                        .toBuilder()
+                    EMPTY_TASK_SUCCESS.toBuilder()
                         .setUniqueName("my:name")
                         .setLog(toByteArraySupplier("my log data"))
                         .build())
@@ -200,11 +192,10 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCreateReportFiles_taskWithoutLog() {
+  void testCreateReportFiles_taskWithoutLog() {
     ImmutableMap<String, String> files =
         getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                 .addTask(EMPTY_TASK_SUCCESS.toBuilder().setUniqueName("my:name").build())
                 .build());
     assertThat(files).doesNotContainKey("logs/my-name.log");
@@ -212,14 +203,12 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCreateReportFiles_taskWithFilledReport() {
+  void testCreateReportFiles_taskWithFilledReport() {
     ImmutableMap<String, String> files =
         getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                 .addTask(
-                    EMPTY_TASK_SUCCESS
-                        .toBuilder()
+                    EMPTY_TASK_SUCCESS.toBuilder()
                         .putReport(
                             "someReport",
                             FilesWithEntryPoint.create(
@@ -234,14 +223,12 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCreateReportFiles_taskWithEmptyReport() {
+  void testCreateReportFiles_taskWithEmptyReport() {
     ImmutableMap<String, String> files =
         getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                 .addTask(
-                    EMPTY_TASK_SUCCESS
-                        .toBuilder()
+                    EMPTY_TASK_SUCCESS.toBuilder()
                         .putReport(
                             "someReport",
                             FilesWithEntryPoint.create(
@@ -254,14 +241,12 @@ public final class CoverPageGeneratorTest {
   }
 
   @Test
-  public void testCreateReportFiles_taskWithLogAndMultipleReports() {
+  void testCreateReportFiles_taskWithLogAndMultipleReports() {
     ImmutableMap<String, String> files =
         getGeneratedFiles(
-            EMPTY_PROJECT
-                .toBuilder()
+            EMPTY_PROJECT.toBuilder()
                 .addTask(
-                    EMPTY_TASK_SUCCESS
-                        .toBuilder()
+                    EMPTY_TASK_SUCCESS.toBuilder()
                         .setUniqueName("my:name")
                         .setLog(toByteArraySupplier("log data"))
                         .putReport(

buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java

@@ -23,6 +23,9 @@ import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath;
 import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs;
 import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.createDirectories;
+import static java.nio.file.Files.createDirectory;
+import static java.nio.file.Files.createFile;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -36,22 +39,20 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Optional;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 
 /** Tests for {@link GcsPluginUtilsTest} */
-@RunWith(JUnit4.class)
-public final class GcsPluginUtilsTest {
+final class GcsPluginUtilsTest {
 
   private static final Joiner filenameJoiner = Joiner.on(File.separator);
 
-  @Rule public final TemporaryFolder folder = new TemporaryFolder();
+  @SuppressWarnings("WeakerAccess")
+  @TempDir
+  Path tmpDir;
 
   @Test
-  public void testGetContentType_knownTypes() {
+  void testGetContentType_knownTypes() {
     assertThat(getContentType("path/to/file.html")).isEqualTo("text/html");
     assertThat(getContentType("path/to/file.htm")).isEqualTo("text/html");
     assertThat(getContentType("path/to/file.log")).isEqualTo("text/plain");
@@ -63,12 +64,12 @@ public final class GcsPluginUtilsTest {
   }
 
   @Test
-  public void testGetContentType_unknownTypes() {
+  void testGetContentType_unknownTypes() {
     assertThat(getContentType("path/to/file.unknown")).isEqualTo("application/octet-stream");
   }
 
   @Test
-  public void testUploadFileToGcs() {
+  void testUploadFileToGcs() {
     Storage storage = mock(Storage.class);
     uploadFileToGcs(
         storage, "my-bucket", Paths.get("my", "filename.txt"), toByteArraySupplier("my data"));
@@ -82,7 +83,7 @@ public final class GcsPluginUtilsTest {
   }
 
   @Test
-  public void testUploadFilesToGcsMultithread() {
+  void testUploadFilesToGcsMultithread() {
     Storage storage = mock(Storage.class);
     uploadFilesToGcsMultithread(
         storage,
@@ -121,21 +122,21 @@ public final class GcsPluginUtilsTest {
   }
 
   @Test
-  public void testToByteArraySupplier_string() {
+  void testToByteArraySupplier_string() {
     assertThat(toByteArraySupplier("my string").get()).isEqualTo("my string".getBytes(UTF_8));
   }
 
   @Test
-  public void testToByteArraySupplier_stringSupplier() {
+  void testToByteArraySupplier_stringSupplier() {
     assertThat(toByteArraySupplier(() -> "my string").get()).isEqualTo("my string".getBytes(UTF_8));
   }
 
   @Test
-  public void testToByteArraySupplier_file() throws Exception {
-    folder.newFolder("arbitrary");
-    File file = folder.newFile("arbitrary/file.txt");
-    Files.write(file.toPath(), "some data".getBytes(UTF_8));
-    assertThat(toByteArraySupplier(file).get()).isEqualTo("some data".getBytes(UTF_8));
+  void testToByteArraySupplier_file() throws Exception {
+    Path dir = createDirectory(tmpDir.resolve("arbitrary"));
+    Path file = createFile(dir.resolve("file.txt"));
+    Files.write(file, "some data".getBytes(UTF_8));
+    assertThat(toByteArraySupplier(file.toFile()).get()).isEqualTo("some data".getBytes(UTF_8));
   }
 
   private ImmutableMap<String, String> readAllFiles(FilesWithEntryPoint reportFiles) {
@@ -147,16 +148,16 @@ public final class GcsPluginUtilsTest {
   }
 
   @Test
-  public void testCreateReportFiles_destinationIsFile() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    folder.newFolder("my", "root", "some", "path");
-    File destination = folder.newFile("my/root/some/path/file.txt");
-    Files.write(destination.toPath(), "some data".getBytes(UTF_8));
+  void testCreateReportFiles_destinationIsFile() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path somePath = createDirectories(root.resolve("some/path"));
+    Path destination = createFile(somePath.resolve("file.txt"));
+    Files.write(destination, "some data".getBytes(UTF_8));
     // Since the entry point is obvious here - any hint given is just ignored.
-    File ignoredHint = folder.newFile("my/root/ignored.txt");
+    File ignoredHint = createFile(root.resolve("ignored.txt")).toFile();
 
     FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root);
 
     assertThat(files.entryPoint().toString())
         .isEqualTo(filenameJoiner.join("some", "path", "file.txt"));
@@ -165,13 +166,13 @@ public final class GcsPluginUtilsTest {
   }
 
   @Test
-  public void testCreateReportFiles_destinationDoesntExist() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
+  void testCreateReportFiles_destinationDoesntExist() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
     File destination = root.resolve("non/existing.txt").toFile();
     assertThat(destination.isFile()).isFalse();
     assertThat(destination.isDirectory()).isFalse();
-    // Since there are not files, any hint given is obvioulsy wrong and will be ignored.
-    File ignoredHint = folder.newFile("my/root/ignored.txt");
+    // Since there are no files, any hint given is obviously wrong and will be ignored.
+    File ignoredHint = createFile(root.resolve("ignored.txt")).toFile();
 
     FilesWithEntryPoint files =
         readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
@@ -181,34 +182,33 @@ public final class GcsPluginUtilsTest {
   }
 
   @Test
-  public void testCreateReportFiles_noFiles() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
-    folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
-    // Since there are not files, any hint given is obvioulsy wrong and will be ignored.
-    File ignoredHint = folder.newFile("my/root/ignored.txt");
+  void testCreateReportFiles_noFiles() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
+    createDirectories(destination.resolve("a/b"));
+    createDirectory(destination.resolve("c"));
+    // Since there are not files, any hint given is obviously wrong and will be ignored.
+    File ignoredHint = createFile(root.resolve("ignored.txt")).toFile();
 
     FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root);
 
     assertThat(files.entryPoint().toString()).isEqualTo(filenameJoiner.join("some", "path"));
     assertThat(files.files()).isEmpty();
   }
 
   @Test
-  public void testCreateReportFiles_oneFile() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
-    folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
-    Files.write(
-        folder.newFile("my/root/some/path/a/file.txt").toPath(), "some data".getBytes(UTF_8));
+  void testCreateReportFiles_oneFile() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
+    createDirectories(destination.resolve("a/b"));
+    createDirectory(destination.resolve("c"));
+    Files.write(createFile(destination.resolve("a/file.txt")), "some data".getBytes(UTF_8));
     // Since the entry point is obvious here - any hint given is just ignored.
-    File ignoredHint = folder.newFile("my/root/ignored.txt");
+    File ignoredHint = createFile(root.resolve("ignored.txt")).toFile();
 
     FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root);
 
     assertThat(files.entryPoint().toString())
         .isEqualTo(filenameJoiner.join("some", "path", "a", "file.txt"));
@@ -222,22 +222,19 @@ public final class GcsPluginUtilsTest {
    * <p>TODO(guyben): switch to checking zip file instead.
    */
   @Test
-  public void testCreateReportFiles_multipleFiles_noHint() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
-    folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
+  void testCreateReportFiles_multipleFiles_noHint() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
+    createDirectories(destination.resolve("a/b"));
+    createDirectory(destination.resolve("c"));
 
-    Files.write(
-        folder.newFile("my/root/some/path/index.html").toPath(), "some data".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("index.html")), "some data".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8));
 
-    FilesWithEntryPoint files = readFilesWithEntryPoint(destination, Optional.empty(), root);
+    FilesWithEntryPoint files =
+        readFilesWithEntryPoint(destination.toFile(), Optional.empty(), root);
 
     assertThat(files.entryPoint().toString())
         .isEqualTo(filenameJoiner.join("some", "path", "path.zip"));
@@ -251,24 +248,20 @@ public final class GcsPluginUtilsTest {
    * <p>TODO(guyben): switch to checking zip file instead.
    */
   @Test
-  public void testCreateReportFiles_multipleFiles_withBadHint() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
+  void testCreateReportFiles_multipleFiles_withBadHint() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
     // This entry point points to a directory, which isn't an appropriate entry point
-    File badEntryPoint = folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
+    File badEntryPoint = createDirectories(destination.resolve("a/b")).toFile();
+    createDirectory(destination.resolve("c"));
 
-    Files.write(
-        folder.newFile("my/root/some/path/index.html").toPath(), "some data".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("index.html")), "some data".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8));
 
     FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(badEntryPoint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(badEntryPoint), root);
 
     assertThat(files.entryPoint().toString())
         .isEqualTo(filenameJoiner.join("some", "path", "path.zip"));
@@ -277,24 +270,21 @@ public final class GcsPluginUtilsTest {
   }
 
   @Test
-  public void testCreateReportFiles_multipleFiles_withGoodHint() throws Exception {
-    Path root = toNormalizedPath(folder.newFolder("my", "root"));
-    File destination = folder.newFolder("my", "root", "some", "path");
-    folder.newFolder("my", "root", "some", "path", "a", "b");
-    folder.newFolder("my", "root", "some", "path", "c");
+  void testCreateReportFiles_multipleFiles_withGoodHint() throws Exception {
+    Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath());
+    Path destination = createDirectories(root.resolve("some/path"));
+    createDirectories(destination.resolve("a/b"));
+    createDirectory(destination.resolve("c"));
     // The hint is an actual file nested in the destination directory!
-    File goodEntryPoint = folder.newFile("my/root/some/path/index.html");
+    Path goodEntryPoint = createFile(destination.resolve("index.html"));
 
-    Files.write(goodEntryPoint.toPath(), "some data".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
-    Files.write(
-        folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
+    Files.write(goodEntryPoint, "some data".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8));
+    Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8));
 
     FilesWithEntryPoint files =
-        readFilesWithEntryPoint(destination, Optional.of(goodEntryPoint), root);
+        readFilesWithEntryPoint(destination.toFile(), Optional.of(goodEntryPoint.toFile()), root);
 
     assertThat(files.entryPoint().toString())
         .isEqualTo(filenameJoiner.join("some", "path", "index.html"));

common/build.gradle

@@ -61,8 +61,6 @@ dependencies {
   testingCompile deps['com.google.truth:truth']
   testingCompile deps['io.github.java-diff-utils:java-diff-utils']
 
-  testCompile deps['junit:junit']
   testCompile deps['org.junit.jupiter:junit-jupiter-api']
   testCompile deps['org.junit.jupiter:junit-jupiter-engine']
-  testCompile deps['org.junit.vintage:junit-vintage-engine']
 }

common/gradle/dependency-locks/apt.lockfile

@@ -1,3 +0,0 @@
-# This is a Gradle generated file for dependency locking.
-# Manual edits can break the build and are not advised.
-# This file is expected to be part of source control.

common/gradle/dependency-locks/testApt.lockfile

@@ -1,3 +0,0 @@
-# This is a Gradle generated file for dependency locking.
-# Manual edits can break the build and are not advised.
-# This file is expected to be part of source control.

common/gradle/dependency-locks/testCompile.lockfile

@@ -14,16 +14,15 @@ com.googlecode.java-diff-utils:diffutils:1.3.0
 io.github.java-diff-utils:java-diff-utils:4.0
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 org.apiguardian:apiguardian-api:1.1.0
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -14,16 +14,15 @@ com.googlecode.java-diff-utils:diffutils:1.3.0
 io.github.java-diff-utils:java-diff-utils:4.0
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 org.apiguardian:apiguardian-api:1.1.0
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testRuntime.lockfile

@@ -15,16 +15,15 @@ com.googlecode.java-diff-utils:diffutils:1.3.0
 io.github.java-diff-utils:java-diff-utils:4.0
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 org.apiguardian:apiguardian-api:1.1.0
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -15,16 +15,15 @@ com.googlecode.java-diff-utils:diffutils:1.3.0
 io.github.java-diff-utils:java-diff-utils:4.0
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-junit:junit:4.13
+junit:junit:4.12
 org.apiguardian:apiguardian-api:1.1.0
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testingApt.lockfile

@@ -1,3 +0,0 @@
-# This is a Gradle generated file for dependency locking.
-# Manual edits can break the build and are not advised.
-# This file is expected to be part of source control.

common/src/test/java/google/registry/testing/truth/TextDiffSubjectTest.java

@@ -17,7 +17,7 @@ package google.registry.testing.truth;
 import static com.google.common.io.Resources.getResource;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.testing.truth.TextDiffSubject.assertThat;
-import static org.junit.Assert.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
@@ -25,13 +25,10 @@ import com.google.common.io.Resources;
 import google.registry.testing.truth.TextDiffSubject.DiffFormat;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
+import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link TextDiffSubject}. */
-@RunWith(JUnit4.class)
-public class TextDiffSubjectTest {
+class TextDiffSubjectTest {
 
   private static final String RESOURCE_FOLDER = "google/registry/testing/truth/";
   // Resources for input data.
@@ -44,21 +41,21 @@ public class TextDiffSubjectTest {
       RESOURCE_FOLDER + "text-sidebyside-diff.txt";
 
   @Test
-  public void unifiedDiff_equal() throws IOException {
+  void unifiedDiff_equal() throws IOException {
     assertThat(getResource(ACTUAL_RESOURCE))
         .withDiffFormat(DiffFormat.UNIFIED_DIFF)
         .hasSameContentAs(getResource(ACTUAL_RESOURCE));
   }
 
   @Test
-  public void sideBySideDiff_equal() throws IOException {
+  void sideBySideDiff_equal() throws IOException {
     assertThat(getResource(ACTUAL_RESOURCE))
         .withDiffFormat(DiffFormat.SIDE_BY_SIDE_MARKDOWN)
         .hasSameContentAs(getResource(ACTUAL_RESOURCE));
   }
 
   @Test
-  public void unifedDiff_notEqual() throws IOException {
+  void unifedDiff_notEqual() throws IOException {
     assertThrows(
         AssertionError.class,
         () ->
@@ -68,7 +65,7 @@ public class TextDiffSubjectTest {
   }
 
   @Test
-  public void sideBySideDiff_notEqual() throws IOException {
+  void sideBySideDiff_notEqual() throws IOException {
     assertThrows(
         AssertionError.class,
         () ->
@@ -78,13 +75,13 @@ public class TextDiffSubjectTest {
   }
 
   @Test
-  public void displayed_unifiedDiff_noDiff() throws IOException {
+  void displayed_unifiedDiff_noDiff() throws IOException {
     ImmutableList<String> actual = readAllLinesFromResource(ACTUAL_RESOURCE);
     assertThat(TextDiffSubject.generateUnifiedDiff(actual, actual)).isEqualTo("");
   }
 
   @Test
-  public void displayed_unifiedDiff_hasDiff() throws IOException {
+  void displayed_unifiedDiff_hasDiff() throws IOException {
     ImmutableList<String> actual = readAllLinesFromResource(ACTUAL_RESOURCE);
     ImmutableList<String> expected = readAllLinesFromResource(EXPECTED_RESOURCE);
     String diff = Joiner.on('\n').join(readAllLinesFromResource(UNIFIED_DIFF_RESOURCE));
@@ -92,7 +89,7 @@ public class TextDiffSubjectTest {
   }
 
   @Test
-  public void displayed_sideBySideDiff_hasDiff() throws IOException {
+  void displayed_sideBySideDiff_hasDiff() throws IOException {
     ImmutableList<String> actual = readAllLinesFromResource(ACTUAL_RESOURCE);
     ImmutableList<String> expected = readAllLinesFromResource(EXPECTED_RESOURCE);
     String diff = Joiner.on('\n').join(readAllLinesFromResource(SIDE_BY_SIDE_DIFF_RESOURCE));

common/src/testing/java/google/registry/testing/truth/TextDiffSubject.java

@@ -16,6 +16,7 @@ package google.registry.testing.truth;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.truth.Truth.assertAbout;
+import static com.google.common.truth.Truth.assertWithMessage;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.github.difflib.DiffUtils;
@@ -31,6 +32,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.io.Resources;
 import com.google.common.truth.Fact;
 import com.google.common.truth.FailureMetadata;
+import com.google.common.truth.SimpleSubjectBuilder;
 import com.google.common.truth.Subject;
 import java.io.IOException;
 import java.net.URL;
@@ -68,6 +70,15 @@ public class TextDiffSubject extends Subject {
     this.actual = ImmutableList.copyOf(actual);
   }
 
+  protected TextDiffSubject(FailureMetadata metadata, URL actual) {
+    super(metadata, actual);
+    try {
+      this.actual = ImmutableList.copyOf(Resources.asCharSource(actual, UTF_8).readLines());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   public TextDiffSubject withDiffFormat(DiffFormat format) {
     this.diffFormat = format;
     return this;
@@ -100,6 +111,11 @@ public class TextDiffSubject extends Subject {
     return assertThat(Resources.asCharSource(resourceUrl, UTF_8).readLines());
   }
 
+  public static SimpleSubjectBuilder<TextDiffSubject, URL> assertWithMessageAboutUrlSource(
+      String format, Object... params) {
+    return assertWithMessage(format, params).about(urlFactory());
+  }
+
   private static final Subject.Factory<TextDiffSubject, ImmutableList<String>>
       TEXT_DIFF_SUBJECT_TEXT_FACTORY = TextDiffSubject::new;
 
@@ -107,6 +123,13 @@ public class TextDiffSubject extends Subject {
     return TEXT_DIFF_SUBJECT_TEXT_FACTORY;
   }
 
+  private static final Subject.Factory<TextDiffSubject, URL> TEXT_DIFF_SUBJECT_URL_FACTORY =
+      TextDiffSubject::new;
+
+  public static Subject.Factory<TextDiffSubject, URL> urlFactory() {
+    return TEXT_DIFF_SUBJECT_URL_FACTORY;
+  }
+
   static String generateUnifiedDiff(
       ImmutableList<String> expectedContent, ImmutableList<String> actualContent) {
     Patch<String> diff;

config/nom_build.py

@@ -19,6 +19,7 @@ import argparse
 import attr
 import io
 import os
+import shutil
 import subprocess
 import sys
 from typing import List, Union
@@ -49,15 +50,30 @@ PROPERTIES_HEADER = """\
 # This file defines properties used by the gradle build.  It must be kept in
 # sync with config/nom_build.py.
 #
-# To regenerate, run config/nom_build.py --generate-gradle-properties
+# To regenerate, run ./nom_build --generate-gradle-properties
 #
 # To view property descriptions (which are command line flags for
-# nom_build), run config/nom_build.py --help.
+# nom_build), run ./nom_build --help.
 #
 # DO NOT EDIT THIS FILE BY HAND
 org.gradle.jvmargs=-Xmx1024m
 """
 
+# Help text to be displayed (in addition to the synopsis and flag help, which
+# are displayed automatically).
+HELP_TEXT = """\
+A wrapper around the gradle build that provides the following features:
+
+-   Converts properties into flags to guard against property name spelling errors
+    and to provide help descriptions for all properties.
+-   Provides pseudo-commands (with the ":nom:" prefix) that encapsulate common
+    actions that are difficult to implement in gradle.
+
+Pseudo-commands:
+    :nom:generate_golden_file - regenerates the golden file from the current
+        set of flyway files.
+"""
+
 # Define all of our special gradle properties here.
 PROPERTIES = [
     Property('mavenUrl',
@@ -114,6 +130,11 @@ PROPERTIES = [
     Property('nomulus_version',
              'The version of nomulus to test against in a database '
              'integration test.'),
+    Property('dot_path',
+             'The path to "dot", part of the graphviz package that converts '
+             'a BEAM pipeline to image. Setting this property to empty string '
+             'will disable image generation.',
+             '/usr/bin/dot'),
 ]
 
 GRADLE_FLAGS = [
@@ -251,8 +272,42 @@ def get_root() -> str:
     return cur_dir
 
 
-def main(args):
-    parser = argparse.ArgumentParser('nom_build')
+class Abort(Exception):
+    """Raised to terminate the process with a non-zero error code.
+
+    Parameters are ignored.
+    """
+
+
+def do_pseudo_task(task: str) -> None:
+    root = get_root()
+    if task == ':nom:generate_golden_file':
+        if not subprocess.call([f'{root}/gradlew', ':db:test']):
+            print('\033[33mWARNING:\033[0m Golden schema appears to be '
+                  'up-to-date.  If you are making schema changes,  be sure to '
+                  'add a flyway file for them.')
+            return
+        print('\033[33mWARNING:\033[0m Ignore the above failure, it is '
+              'expected.')
+
+        # Copy the new schema into place.
+        shutil.copy(f'{root}/db/build/resources/test/testcontainer/'
+                    'mount/dump.txt',
+                    f'{root}/db/src/main/resources/sql/schema/'
+                    'nomulus.golden.sql')
+
+        if subprocess.call([f'{root}/gradlew', ':db:test']):
+            print('\033[31mERROR:\033[0m Golden file test failed after '
+                  'copying schema.  Please check your flyway files.')
+            raise Abort()
+    else:
+        print(f'\033[31mERROR:\033[0m Unknown task {task}')
+        raise Abort()
+
+
+def main(args) -> int:
+    parser = argparse.ArgumentParser('nom_build', description=HELP_TEXT,
+                                     formatter_class=argparse.RawTextHelpFormatter)
     for prop in PROPERTIES:
         parser.add_argument('--' + prop.name, default=prop.default,
                             help=prop.desc)
@@ -291,7 +346,7 @@ def main(args):
     if args.generate_gradle_properties:
         with open(f'{root}/gradle.properties', 'w') as dst:
             dst.write(gradle_properties)
-        return
+        return 0
 
     # Verify that the gradle properties file is what we expect it to be.
     with open(f'{root}/gradle.properties') as src:
@@ -316,12 +371,39 @@ def main(args):
             if flag.has_arg:
                 gradle_command.append(arg_val)
 
+    # See if there are any special ":nom:" pseudo-tasks specified.
+    got_non_pseudo_tasks = False
+    for arg in args.non_flag_args[1:]:
+        if arg.startswith(':nom:'):
+            if got_non_pseudo_tasks:
+                # We can't currently deal with the situation of gradle tasks
+                # before pseudo-tasks.  This could be implemented by invoking
+                # gradle for only the set of gradle tasks before the pseudo
+                # task, but that's overkill for now.
+                print(f'\033[31mERROR:\033[0m Pseudo task ({arg}) must be '
+                      'specified prior to all actual gradle tasks.  Aborting.')
+                return 1
+            do_pseudo_task(arg)
+        else:
+            got_non_pseudo_tasks = True
+    non_flag_args = [
+        arg for arg in args.non_flag_args[1:] if not arg.startswith(':nom:')]
+
+    if not non_flag_args:
+        if not got_non_pseudo_tasks:
+            print('\033[33mWARNING:\033[0m No tasks specified.  Not '
+                  'doing anything')
+        return 0
+
     # Add the non-flag args (we exclude the first, which is the command name
     # itself) and run.
-    gradle_command.extend(args.non_flag_args[1:])
-    subprocess.call(gradle_command)
+    gradle_command.extend(non_flag_args)
+    return subprocess.call(gradle_command)
 
 
 if __name__ == '__main__':
-    main(sys.argv)
+    try:
+        sys.exit(main(sys.argv))
+    except Abort as ex:
+        sys.exit(1)
 

config/nom_build_test.py

@@ -14,6 +14,7 @@
 
 import io
 import os
+import shutil
 import unittest
 from unittest import mock
 import nom_build
@@ -67,6 +68,7 @@ class MyTest(unittest.TestCase):
             mock.patch.object(nom_build, 'print', self.print_fake).start())
 
         self.call_mock = mock.patch.object(subprocess, 'call').start()
+        self.copy_mock = mock.patch.object(shutil, 'copy').start()
 
         self.file_contents = {
             # Prefil with the actual file contents.
@@ -92,17 +94,32 @@ class MyTest(unittest.TestCase):
 
     def test_no_args(self):
         nom_build.main(['nom_build'])
-        self.assertEqual(self.printed, [])
-        self.call_mock.assert_called_with([GRADLEW])
+        self.assertEqual(self.printed,
+                         ['\x1b[33mWARNING:\x1b[0m No tasks specified.  Not '
+                         'doing anything'])
 
     def test_property_calls(self):
-        nom_build.main(['nom_build', '--testFilter=foo'])
-        self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo'])
+        nom_build.main(['nom_build', 'task-name', '--testFilter=foo'])
+        self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo',
+                                          'task-name'])
 
     def test_gradle_flags(self):
-        nom_build.main(['nom_build', '-d', '-b', 'foo'])
+        nom_build.main(['nom_build', 'task-name', '-d', '-b', 'foo'])
         self.call_mock.assert_called_with([GRADLEW, '--build-file', 'foo',
-                                          '--debug'])
+                                          '--debug', 'task-name'])
+
+    def test_generate_golden_file(self):
+        self.call_mock.side_effect = [1, 0]
+        nom_build.main(['nom_build', ':nom:generate_golden_file'])
+        self.call_mock.assert_has_calls([
+            mock.call([GRADLEW, ':db:test']),
+            mock.call([GRADLEW, ':db:test'])
+        ])
+
+    def test_generate_golden_file_nofail(self):
+        self.call_mock.return_value = 0
+        nom_build.main(['nom_build', ':nom:generate_golden_file'])
+        self.call_mock.assert_has_calls([mock.call([GRADLEW, ':db:test'])])
 
 unittest.main()
 

config/presubmits.py

@@ -22,7 +22,7 @@ import sys
 import re
 
 # We should never analyze any generated files
-UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/"}
+UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/"}
 # We can't rely on CI to have the Enum package installed so we do this instead.
 FORBIDDEN = 1
 REQUIRED = 2
@@ -77,9 +77,9 @@ PRESUBMITS = {
     PresubmitCheck(
         r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
         ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
-            ".git", "/build/", "/generated/", "node_modules/",
-            "JUnitBackports.java", "registrar_bin.", "registrar_dbg.",
-            "google-java-format-diff.py",
+            ".git", "/build/", "/generated/", "/generated_tests/",
+            "node_modules/", "JUnitBackports.java", "registrar_bin.",
+            "registrar_dbg.", "google-java-format-diff.py",
             "nomulus.golden.sql", "soyutils_usegoog.js"
         }, REQUIRED):
         "File did not include the license header.",
@@ -93,20 +93,20 @@ PRESUBMITS = {
     PresubmitCheck(
         r".*\bSystem\.(out|err)\.print", "java", {
             "StackdriverDashboardBuilder.java", "/tools/", "/example/",
-            "RegistryTestServerMain.java", "TestServerRule.java",
+            "RegistryTestServerMain.java", "TestServerExtension.java",
             "FlowDocumentationTool.java"
         }):
         "System.(out|err).println is only allowed in tools/ packages. Please "
         "use a logger instead.",
 
-    # ObjectifyService.register is restricted to main/ or AppEngineRule.
+    # ObjectifyService.register is restricted to main/ or AppEngineExtension.
     PresubmitCheck(
         r".*\bObjectifyService\.register", "java", {
             "/build/", "/generated/", "node_modules/", "src/main/",
-            "AppEngineRule.java"
+            "AppEngineExtension.java"
         }):
-      "ObjectifyService.register is not allowed in tests. Please use "
-      "AppengineRule.register instead.",
+      "ObjectifyService.register(...) is not allowed in tests. Please use "
+      "AppEngineExtension.register(...) instead.",
 
     # PostgreSQLContainer instantiation must specify docker tag
     PresubmitCheck(

core/build.gradle

@@ -22,7 +22,7 @@ plugins {
 
 // Path to code generated by ad hoc tasks in this project. A separate path is
 // used for easy inspection.
-def generatedDir = "${project.buildDir}/generated/source/custom/main"
+def generatedDir = "${project.buildDir}/generated/sources/custom/java/main"
 def resourcesDir = "${project.buildDir}/resources/main"
 def screenshotsDir = "${project.buildDir}/screenshots"
 def screenshotsForGoldensDir = "${project.buildDir}/screenshots_for_goldens"
@@ -82,11 +82,6 @@ sourceSets {
   main {
     java {
       srcDirs += generatedDir
-      // Javadoc API is deprecated in Java 11 and removed in Java 12.
-      // TODO(jianglai): re-enable after migrating to the new Javadoc API
-      if ((JavaVersion.current().majorVersion as Integer) >= 11) {
-        exclude 'google/registry/documentation/**'
-      }
     }
     resources {
       exclude '**/*.xjb'
@@ -238,6 +233,7 @@ dependencies {
   compile deps['jline:jline']
   compile deps['joda-time:joda-time']
   compile deps['org.apache.avro:avro']
+  testCompile deps['org.apache.beam:beam-runners-core-construction-java']
   testCompile deps['org.apache.beam:beam-runners-direct-java']
   compile deps['org.apache.beam:beam-runners-google-cloud-dataflow-java']
   compile deps['org.apache.beam:beam-sdks-java-core']
@@ -256,12 +252,13 @@ dependencies {
   compile deps['org.bouncycastle:bcpg-jdk15on']
   testCompile deps['org.bouncycastle:bcpkix-jdk15on']
   compile deps['org.bouncycastle:bcprov-jdk15on']
+  testCompile deps['com.fasterxml.jackson.core:jackson-databind']
   runtime deps['org.glassfish.jaxb:jaxb-runtime']
   compile deps['org.hibernate:hibernate-core']
   compile deps['org.joda:joda-money']
   compile deps['org.json:json']
   testCompile deps['org.mortbay.jetty:jetty']
-  runtimeOnly deps['org.postgresql:postgresql']
+  compile deps['org.postgresql:postgresql']
   testCompile deps['org.seleniumhq.selenium:selenium-api']
   testCompile deps['org.seleniumhq.selenium:selenium-chrome-driver']
   testCompile deps['org.seleniumhq.selenium:selenium-java']
@@ -272,7 +269,6 @@ dependencies {
   compile deps['org.testcontainers:postgresql']
   testCompile deps['org.testcontainers:selenium']
   testCompile deps['org.testcontainers:testcontainers']
-  testCompile deps['pl.pragmatists:JUnitParams']
   compile deps['xerces:xmlParserAPIs']
   compile deps['xpp3:xpp3']
   // This dependency must come after javax.mail:mail as it would otherwise
@@ -312,9 +308,10 @@ dependencies {
   testCompile deps['org.junit.jupiter:junit-jupiter-api']
   testCompile deps['org.junit.jupiter:junit-jupiter-engine']
   testCompile deps['org.junit.jupiter:junit-jupiter-migrationsupport']
+  testCompile deps['org.junit.jupiter:junit-jupiter-params']
+  testCompile deps['org.junit-pioneer:junit-pioneer']
   testCompile deps['org.junit.platform:junit-platform-runner']
   testCompile deps['org.junit.platform:junit-platform-suite-api']
-  testCompile deps['org.junit.vintage:junit-vintage-engine']
   testCompile deps['org.mockito:mockito-core']
   testCompile deps['org.mockito:mockito-junit-jupiter']
   runtime deps['org.postgresql:postgresql']
@@ -407,6 +404,9 @@ task jaxbToJava {
         }
       }
     }
+    execInBash(
+        'find . -name *.java -exec sed -i /\\*\\ \\<p\\>\\$/d {} +',
+        generatedDir)
   }
 }
 
@@ -705,6 +705,10 @@ task fragileTest(type: FilteringTest) {
 
   // Run every test class in a freshly started process.
   forkEvery 1
+
+  doFirst {
+    new File(screenshotsDir).deleteDir()
+  }
 }
 
 task outcastTest(type: FilteringTest) {
@@ -822,6 +826,30 @@ createToolTask(
     'google.registry.tools.DevTool',
     sourceSets.nonprod)
 
+project.tasks.create('initSqlPipeline', JavaExec) {
+  main = 'google.registry.beam.initsql.InitSqlPipeline'
+
+  doFirst {
+    getToolArgsList().ifPresent {
+      args it
+    }
+
+    def isDirectRunner =
+        args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
+    // The dependency containing DirectRunner is intentionally excluded from the
+    // production binary, so that it won't be chosen by mistake: we definitely do
+    // not want to use it for the real jobs, yet DirectRunner is the default if
+    // the user forgets to override it.
+    // DirectRunner is required for tests and is already on testRuntimeClasspath.
+    // For simplicity, we add testRuntimeClasspath to this task's classpath instead
+    // of defining a new configuration just for the DirectRunner dependency.
+    classpath =
+        isDirectRunner
+            ? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
+            : sourceSets.main.runtimeClasspath
+  }
+}
+
 project.tasks.create('generateSqlSchema', JavaExec) {
   classpath = sourceSets.nonprod.runtimeClasspath
   main = 'google.registry.tools.DevTool'
@@ -849,22 +877,6 @@ task generateGoldenImages(type: FilteringTest) {
 }
 generateGoldenImages.finalizedBy(findGoldenImages)
 
-task flowDocsTool(type: JavaExec) {
-  systemProperty 'test.projectRoot', rootProject.projectRootDir
-  systemProperty 'test.resourcesDir', resourcesDir
-
-  classpath = sourceSets.main.runtimeClasspath
-  main = 'google.registry.documentation.FlowDocumentationTool'
-
-  def arguments = []
-  if (rootProject.flowDocsFile) {
-    arguments << "--output_file=${rootProject.flowDocsFile}"
-  } else {
-    arguments << "--output_file=${rootProject.projectRootDir}/docs/flows.md"
-  }
-  args arguments
-}
-
 task standardTest(type: FilteringTest) {
   includeAllTests()
   exclude fragileTestPatterns
@@ -889,10 +901,6 @@ task standardTest(type: FilteringTest) {
 
   systemProperty 'test.projectRoot', rootProject.projectRootDir
   systemProperty 'test.resourcesDir', resourcesDir
-
-  doFirst {
-    new File(screenshotsDir).deleteDir()
-  }
 }
 
 test {
@@ -966,6 +974,49 @@ task buildToolImage(dependsOn: nomulus, type: Exec) {
   commandLine 'docker', 'build', '-t', 'nomulus-tool', '.'
 }
 
+task generateInitSqlPipelineGraph(type: Test) {
+  include "**/InitSqlPipelineGraphTest.*"
+  testNameIncludePatterns = ["**createPipeline_compareGraph"]
+  ignoreFailures = true
+}
+
+task updateInitSqlPipelineGraph(type: Copy) {
+  def graphRelativePath = 'google/registry/beam/initsql/'
+  from ("${projectDir}/build/resources/test/${graphRelativePath}") {
+    include 'pipeline_curr.dot'
+    rename 'curr', 'golden'
+  }
+  into "src/test/resources/${graphRelativePath}"
+
+  dependsOn generateInitSqlPipelineGraph
+
+  doLast {
+    if (com.google.common.base.Strings.isNullOrEmpty(project.dot_path)) {
+      getLogger().info('Property dot_path is null. Not creating image for pipeline graph.')
+    }
+    def dotPath = project.dot_path
+    if (!new File(dotPath).exists()) {
+      throw new RuntimeException(
+          """\
+              ${dotPath} not found. Make sure graphviz is installed
+              and the dot_path property is set correctly."""
+              .stripIndent())
+    }
+    def goldenGraph = "src/test/resources/${graphRelativePath}/pipeline_golden.dot"
+    def goldenImage = "src/test/resources/${graphRelativePath}/pipeline_golden.png"
+    def cmd = "${dotPath} -Tpng -o \"${goldenImage}\" \"${goldenGraph}\""
+    try {
+      rootProject.ext.execInBash(cmd, projectDir)
+    } catch (Throwable throwable) {
+      throw new RuntimeException(
+          """\
+              Failed to generate golden image with command ${cmd}
+              Error: ${throwable.getMessage()}
+          """)
+    }
+  }
+}
+
 // Build the devtool jar.
 createUberJar(
     'devtool',

core/gradle/dependency-locks/apt.lockfile

@@ -1,3 +0,0 @@
-# This is a Gradle generated file for dependency locking.
-# Manual edits can break the build and are not advised.
-# This file is expected to be part of source control.

core/gradle/dependency-locks/compile.lockfile

@@ -228,6 +228,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -223,6 +223,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2

core/gradle/dependency-locks/nonprodApt.lockfile

@@ -1,3 +0,0 @@
-# This is a Gradle generated file for dependency locking.
-# Manual edits can break the build and are not advised.
-# This file is expected to be part of source control.

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -228,6 +228,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -226,6 +226,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2

core/gradle/dependency-locks/testApt.lockfile

@@ -1,3 +0,0 @@
-# This is a Gradle generated file for dependency locking.
-# Manual edits can break the build and are not advised.
-# This file is expected to be part of source control.

core/gradle/dependency-locks/testCompile.lockfile

@@ -243,15 +243,16 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
 org.junit.platform:junit-platform-runner:1.6.2
 org.junit.platform:junit-platform-suite-api:1.6.2
-org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
 org.mockito:mockito-core:3.3.3
@@ -265,6 +266,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
@@ -291,6 +293,5 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
-pl.pragmatists:JUnitParams:1.1.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -241,15 +241,16 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
 org.junit.platform:junit-platform-runner:1.6.2
 org.junit.platform:junit-platform-suite-api:1.6.2
-org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
 org.mockito:mockito-core:3.3.3
@@ -263,6 +264,7 @@ org.ow2.asm:asm-commons:7.1
 org.ow2.asm:asm-tree:8.0.1
 org.ow2.asm:asm-util:8.0.1
 org.ow2.asm:asm:8.0.1
+org.postgresql:postgresql:42.2.14
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
@@ -289,6 +291,5 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
-pl.pragmatists:JUnitParams:1.1.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testRuntime.lockfile

@@ -246,15 +246,16 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
 org.junit.platform:junit-platform-runner:1.6.2
 org.junit.platform:junit-platform-suite-api:1.6.2
-org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
 org.mockito:mockito-core:3.3.3
@@ -295,6 +296,5 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
-pl.pragmatists:JUnitParams:1.1.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -246,15 +246,16 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
 org.junit.platform:junit-platform-runner:1.6.2
 org.junit.platform:junit-platform-suite-api:1.6.2
-org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
 org.mockito:mockito-core:3.3.3
@@ -296,6 +297,5 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
-pl.pragmatists:JUnitParams:1.1.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/karma.conf.js

@@ -39,7 +39,7 @@ module.exports = function(config) {
         included: false
       },
       {
-        pattern: 'core/build/generated/source/custom/main/**/*.soy.js',
+        pattern: 'core/build/generated/sources/custom/java/main/**/*.soy.js',
         included: false
       },
       {
@@ -65,7 +65,7 @@ module.exports = function(config) {
       'node_modules/google-closure-library/closure/goog/base.js': ['closure'],
       'node_modules/google-closure-library/closure/**/*.js': ['closure'],
       'core/src/*/javascript/**/*.js': ['closure'],
-      'core/build/generated/source/custom/main/**/*.soy.js': ['closure'],
+      'core/build/generated/sources/custom/java/main/**/*.soy.js': ['closure'],
     },
     proxies: {
       "/assets/": "/base/core/build/resources/main/google/registry/ui/assets/"

core/src/main/java/google/registry/backup/AppEngineEnvironment.java

@@ -22,21 +22,35 @@ import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 
 /**
- * Sets up a placeholder {@link Environment} on a non-AppEngine platform so that Datastore Entities
- * can be converted from/to Objectify entities. See {@code DatastoreEntityExtension} in test source
- * for more information.
+ * Sets up a fake {@link Environment} so that the following operations can be performed without the
+ * Datastore service:
+ *
+ * <ul>
+ *   <li>Create Objectify {@code Keys}.
+ *   <li>Instantiate Objectify objects.
+ *   <li>Convert Datastore {@code Entities} to their corresponding Objectify objects.
+ * </ul>
+ *
+ * <p>User has the option to specify their desired {@code appId} string, which forms part of an
+ * Objectify {@code Key} and is included in the equality check. This feature makes it easy to
+ * compare a migrated object in SQL with the original in Objectify.
+ *
+ * <p>Note that conversion from Objectify objects to Datastore {@code Entities} still requires the
+ * Datastore service.
  */
 public class AppEngineEnvironment implements Closeable {
 
-  private static final Environment PLACEHOLDER_ENV = createAppEngineEnvironment();
-
   private boolean isPlaceHolderNeeded;
 
   public AppEngineEnvironment() {
+    this("PlaceholderAppId");
+  }
+
+  public AppEngineEnvironment(String appId) {
     isPlaceHolderNeeded = ApiProxy.getCurrentEnvironment() == null;
     // isPlaceHolderNeeded may be true when we are invoked in a test with AppEngineRule.
     if (isPlaceHolderNeeded) {
-      ApiProxy.setEnvironmentForCurrentThread(PLACEHOLDER_ENV);
+      ApiProxy.setEnvironmentForCurrentThread(createAppEngineEnvironment(appId));
     }
   }
 
@@ -48,7 +62,7 @@ public class AppEngineEnvironment implements Closeable {
   }
 
   /** Returns a placeholder {@link Environment} that can return hardcoded AppId and Attributes. */
-  private static Environment createAppEngineEnvironment() {
+  private static Environment createAppEngineEnvironment(String appId) {
     return (Environment)
         Proxy.newProxyInstance(
             Environment.class.getClassLoader(),
@@ -56,7 +70,7 @@ public class AppEngineEnvironment implements Closeable {
             (Object proxy, Method method, Object[] args) -> {
               switch (method.getName()) {
                 case "getAppId":
-                  return "PlaceholderAppId";
+                  return appId;
                 case "getAttributes":
                   return ImmutableMap.<String, Object>of();
                 default:

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -169,10 +169,12 @@ public final class AsyncTaskEnqueuer {
         lock.getRelockDuration().isPresent(),
         "Lock with ID %s not configured for relock",
         lock.getRevisionId());
+    String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
     addTaskToQueueWithRetry(
         asyncActionsPushQueue,
         TaskOptions.Builder.withUrl(RelockDomainAction.PATH)
             .method(Method.POST)
+            .header("Host", backendHostname)
             .param(
                 RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
                 String.valueOf(lock.getRevisionId()))

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -532,7 +532,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
           .setKey(resourceKey)
-          .setLastUpdateTime(resource.getUpdateAutoTimestamp().getTimestamp())
+          .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(
                   params.get(PARAM_REQUESTING_CLIENT_ID), "Requesting client id not specified"))

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -319,13 +319,13 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
       HostResource host =
           checkNotNull(ofy().load().key(hostKey).now(), "Host to refresh doesn't exist");
       boolean isHostDeleted =
-          isDeleted(host, latestOf(now, host.getUpdateAutoTimestamp().getTimestamp()));
+          isDeleted(host, latestOf(now, host.getUpdateTimestamp().getTimestamp()));
       if (isHostDeleted) {
         logger.atInfo().log("Host %s is already deleted, not refreshing DNS.", hostKey);
       }
       return new AutoValue_RefreshDnsOnHostRenameAction_DnsRefreshRequest.Builder()
           .setHostKey(hostKey)
-          .setLastUpdateTime(host.getUpdateAutoTimestamp().getTimestamp())
+          .setLastUpdateTime(host.getUpdateTimestamp().getTimestamp())
           .setRequestedTime(
               DateTime.parse(
                   checkNotNull(params.get(PARAM_REQUESTED_TIME), "Requested time not specified")))

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -32,7 +32,7 @@ public final class BackupPaths {
   private BackupPaths() {}
 
   private static final String WILDCARD_CHAR = "*";
-  private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/input-%s";
+  private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/output-%s";
 
   public static final String COMMIT_LOG_NAME_PREFIX = "commit_diff_until_";
   private static final String COMMIT_LOG_PATTERN_TEMPLATE = "%s/" + COMMIT_LOG_NAME_PREFIX + "*";

core/src/main/java/google/registry/beam/initsql/BeamJpaModule.java

@@ -19,7 +19,6 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
 
 import com.google.common.base.Splitter;
-import dagger.Binds;
 import dagger.Component;
 import dagger.Lazy;
 import dagger.Module;
@@ -32,10 +31,6 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.JdbcJpaTm;
 import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
-import google.registry.util.Clock;
-import google.registry.util.Sleeper;
-import google.registry.util.SystemClock;
-import google.registry.util.SystemSleeper;
 import google.registry.util.UtilsModule;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -59,37 +54,39 @@ public class BeamJpaModule {
 
   private static final String GCS_SCHEME = "gs://";
 
-  @Nullable private final String credentialFilePath;
+  @Nullable private final String sqlAccessInfoFile;
+  @Nullable private final String cloudKmsProjectId;
 
   /**
    * Constructs a new instance of {@link BeamJpaModule}.
    *
    * <p>Note: it is an unfortunately necessary antipattern to check for the validity of
-   * credentialFilePath in {@link #provideCloudSqlAccessInfo} rather than in the constructor.
+   * sqlAccessInfoFile in {@link #provideCloudSqlAccessInfo} rather than in the constructor.
    * Unfortunately, this is a restriction imposed upon us by Dagger. Specifically, because we use
    * this in at least one 1 {@link google.registry.tools.RegistryTool} command(s), it must be
    * instantiated in {@code google.registry.tools.RegistryToolComponent} for all possible commands;
    * Dagger doesn't permit it to ever be null. For the vast majority of commands, it will never be
    * used (so a null credential file path is fine in those cases).
    *
-   * @param credentialFilePath the path to a Cloud SQL credential file. This must refer to either a
+   * @param sqlAccessInfoFile the path to a Cloud SQL credential file. This must refer to either a
    *     real encrypted file on GCS as returned by {@link
    *     BackupPaths#getCloudSQLCredentialFilePatterns} or an unencrypted file on local filesystem
    *     with credentials to a test database.
    */
-  public BeamJpaModule(@Nullable String credentialFilePath) {
-    this.credentialFilePath = credentialFilePath;
+  public BeamJpaModule(@Nullable String sqlAccessInfoFile, @Nullable String cloudKmsProjectId) {
+    this.sqlAccessInfoFile = sqlAccessInfoFile;
+    this.cloudKmsProjectId = cloudKmsProjectId;
   }
 
   /** Returns true if the credential file is on GCS (and therefore expected to be encrypted). */
   private boolean isCloudSqlCredential() {
-    return credentialFilePath.startsWith(GCS_SCHEME);
+    return sqlAccessInfoFile.startsWith(GCS_SCHEME);
   }
 
   @Provides
   @Singleton
   SqlAccessInfo provideCloudSqlAccessInfo(Lazy<CloudSqlCredentialDecryptor> lazyDecryptor) {
-    checkArgument(!isNullOrEmpty(credentialFilePath), "Null or empty credentialFilePath");
+    checkArgument(!isNullOrEmpty(sqlAccessInfoFile), "Null or empty credentialFilePath");
     String line = readOnlyLineFromCredentialFile();
     if (isCloudSqlCredential()) {
       line = lazyDecryptor.get().decrypt(line);
@@ -106,7 +103,7 @@ public class BeamJpaModule {
 
   String readOnlyLineFromCredentialFile() {
     try {
-      ResourceId resourceId = FileSystems.matchSingleFileSpec(credentialFilePath).resourceId();
+      ResourceId resourceId = FileSystems.matchSingleFileSpec(sqlAccessInfoFile).resourceId();
       try (BufferedReader reader =
           new BufferedReader(
               new InputStreamReader(
@@ -146,8 +143,8 @@ public class BeamJpaModule {
 
   @Provides
   @Config("beamCloudKmsProjectId")
-  static String kmsProjectId() {
-    return "domain-registry-dev";
+  String kmsProjectId() {
+    return cloudKmsProjectId;
   }
 
   @Provides
@@ -159,19 +156,10 @@ public class BeamJpaModule {
   @Provides
   @Config("beamHibernateHikariMaximumPoolSize")
   static int getBeamHibernateHikariMaximumPoolSize() {
+    // TODO(weiminyu): make this configurable. Should be equal to number of cores.
     return 4;
   }
 
-  @Module
-  interface BindModule {
-
-    @Binds
-    Sleeper sleeper(SystemSleeper sleeper);
-
-    @Binds
-    Clock clock(SystemClock clock);
-  }
-
   @Singleton
   @Component(
       modules = {

core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java

@@ -0,0 +1,75 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.appengine.api.datastore.Entity;
+import java.util.Objects;
+
+/** Helper for manipulating {@code DomainBase} when migrating from Datastore to SQL database */
+final class DomainBaseUtil {
+
+  private DomainBaseUtil() {}
+
+  /**
+   * Removes {@link google.registry.model.billing.BillingEvent.Recurring}, {@link
+   * google.registry.model.poll.PollMessage PollMessages} and {@link
+   * google.registry.model.host.HostResource name servers} from a Datastore {@link Entity} that
+   * represents an Ofy {@link google.registry.model.domain.DomainBase}. This breaks the cycle of
+   * foreign key constraints between these entity kinds, allowing {@code DomainBases} to be inserted
+   * into the SQL database. See {@link InitSqlPipeline} for a use case, where the full {@code
+   * DomainBases} are written again during the last stage of the pipeline.
+   *
+   * <p>The returned object may be in bad state. Specifically, {@link
+   * google.registry.model.eppcommon.StatusValue#INACTIVE} is not added after name servers are
+   * removed. This only impacts tests.
+   *
+   * <p>This operation is performed on an Datastore {@link Entity} instead of Ofy Java object
+   * because Objectify requires access to a Datastore service when converting an Ofy object to a
+   * Datastore {@code Entity}. If we insist on working with Objectify objects, we face a few
+   * unsatisfactory options:
+   *
+   * <ul>
+   *   <li>Connect to our production Datastore, which incurs unnecessary security and code health
+   *       risk.
+   *   <li>Connect to a separate real Datastore instance, which is a waster and overkill.
+   *   <li>Use an in-memory test Datastore, which is a project health risk in that the test
+   *       Datastore would be added to Nomulus' production binary unless we create a separate
+   *       project for this pipeline.
+   * </ul>
+   *
+   * <p>Given our use case, operating on Datastore entities is the best option.
+   *
+   * @throws IllegalArgumentException if input does not represent a DomainBase
+   */
+  static Entity removeBillingAndPollAndHosts(Entity domainBase) {
+    checkNotNull(domainBase, "domainBase");
+    checkArgument(
+        Objects.equals(domainBase.getKind(), "DomainBase"),
+        "Expecting DomainBase, got %s",
+        domainBase.getKind());
+    Entity clone = domainBase.clone();
+    clone.removeProperty("autorenewBillingEvent");
+    clone.removeProperty("autorenewPollMessage");
+    clone.removeProperty("deletePollMessage");
+    clone.removeProperty("nsHosts");
+    domainBase.getProperties().keySet().stream()
+        .filter(s -> s.startsWith("transferData."))
+        .forEach(s -> clone.removeProperty(s));
+    return clone;
+  }
+}

core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java

@@ -0,0 +1,244 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.AppEngineEnvironment;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.BeamJpaModule.JpaTransactionManagerComponent;
+import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registry.Registry;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.Wait;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+
+/**
+ * A BEAM pipeline that populates a SQL database with data from a Datastore backup.
+ *
+ * <p>This pipeline migrates EPP resources and related entities that cross-reference each other. To
+ * avoid violating foreign key constraints, writes to SQL are ordered by entity kinds. In addition,
+ * the {@link DomainBase} kind is written twice (see details below). The write order is presented
+ * below. Although some kinds can be written concurrently, e.g. {@code ContactResource} and {@code
+ * RegistrarContact}, we do not expect any performance benefit since the limiting resource is the
+ * number of JDBC connections. Google internal users may refer to <a
+ * href="http://go/registry-r3-init-sql">the design doc</a> for more information.
+ *
+ * <ol>
+ *   <li>{@link Registry}: Assumes that {@code PremiumList} and {@code ReservedList} have been set
+ *       up in the SQL database.
+ *   <li>{@link Registrar}: Logically depends on {@code Registry}, Foreign key not modeled yet.
+ *   <li>{@link ContactResource}: references {@code Registrar}
+ *   <li>{@link RegistrarContact}: references {@code Registrar}.
+ *   <li>Cleansed {@link DomainBase}: with references to {@code BillingEvent}, {@code Recurring},
+ *       {@code Cancellation} and {@code HostResource} removed, still references {@code Registrar}
+ *       and {@code ContactResource}. The removal breaks circular Foreign Key references.
+ *   <li>{@link HostResource}: references {@code DomainBase}.
+ *   <li>{@link HistoryEntry}: maps to one of three SQL entity types and may reference {@code
+ *       Registrar}, {@code ContactResource}, {@code HostResource}, and {@code DomainBase}.
+ *   <li>{@link AllocationToken}: references {@code HistoryEntry}.
+ *   <li>{@link BillingEvent.Recurring}: references {@code Registrar}, {@code DomainBase} and {@code
+ *       HistoryEntry}.
+ *   <li>{@link BillingEvent.OneTime}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       BillingEvent.Recurring}, {@code HistoryEntry} and {@code AllocationToken}.
+ *   <li>{@link BillingEvent.Modification}: SQL model TBD. Will reference {@code Registrar}, {@code
+ *       DomainBase} and {@code BillingEvent.OneTime}.
+ *   <li>{@link BillingEvent.Cancellation}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       BillingEvent.Recurring}, {@code BillingEvent.OneTime}, and {@code HistoryEntry}.
+ *   <li>{@link PollMessage}: references {@code Registrar}, {@code DomainBase}, {@code
+ *       ContactResource}, {@code HostResource}, and {@code HistoryEntry}.
+ *   <li>{@link DomainBase}, original copy from Datastore.
+ * </ol>
+ */
+public class InitSqlPipeline implements Serializable {
+
+  /**
+   * Datastore kinds to be written to the SQL database before the cleansed version of {@link
+   * DomainBase}.
+   */
+  // TODO(weiminyu): include Registry.class when it is modeled in JPA.
+  private static final ImmutableList<Class<?>> PHASE_ONE_ORDERED =
+      ImmutableList.of(Registrar.class, ContactResource.class);
+
+  /**
+   * Datastore kinds to be written to the SQL database after the cleansed version of {@link
+   * DomainBase}.
+   *
+   * <p>The following entities are missing from the list:
+   *
+   * <ul>
+   *   <li>Those not modeled in JPA yet, e.g., {@code BillingEvent.Modification}.
+   *   <li>Those waiting for sanitation, e.g., {@code HistoryEntry}, which would have duplicate keys
+   *       after converting to SQL model.
+   *   <li>Those that have foreign key constraints on the above.
+   * </ul>
+   */
+  // TODO(weiminyu): add more entities when available.
+  private static final ImmutableList<Class<?>> PHASE_TWO_ORDERED =
+      ImmutableList.of(HostResource.class);
+
+  private final InitSqlPipelineOptions options;
+
+  private final Pipeline pipeline;
+
+  private final SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager>
+      jpaGetter;
+
+  InitSqlPipeline(InitSqlPipelineOptions options) {
+    this.options = options;
+    pipeline = Pipeline.create(options);
+    jpaGetter = JpaTransactionManagerComponent::cloudSqlJpaTransactionManager;
+  }
+
+  @VisibleForTesting
+  InitSqlPipeline(InitSqlPipelineOptions options, Pipeline pipeline) {
+    this.options = options;
+    this.pipeline = pipeline;
+    jpaGetter = JpaTransactionManagerComponent::localDbJpaTransactionManager;
+  }
+
+  public PipelineResult run() {
+    setupPipeline();
+    return pipeline.run();
+  }
+
+  @VisibleForTesting
+  void setupPipeline() {
+    PCollectionTuple datastoreSnapshot =
+        pipeline.apply(
+            "Load Datastore snapshot",
+            Transforms.loadDatastoreSnapshot(
+                options.getDatastoreExportDir(),
+                options.getCommitLogDir(),
+                DateTime.parse(options.getCommitLogStartTimestamp()),
+                DateTime.parse(options.getCommitLogEndTimestamp()),
+                ImmutableSet.<String>builder()
+                    .add("DomainBase")
+                    .addAll(toKindStrings(PHASE_ONE_ORDERED))
+                    .addAll(toKindStrings(PHASE_TWO_ORDERED))
+                    .build()));
+
+    // Set up the pipeline to write entity kinds from PHASE_ONE_ORDERED to SQL. Return a object
+    // that signals the completion of the phase.
+    PCollection<Void> blocker =
+        scheduleOnePhaseWrites(datastoreSnapshot, PHASE_ONE_ORDERED, Optional.empty(), null);
+    blocker =
+        writeToSql(
+            "DomainBase without circular foreign keys",
+            removeDomainBaseForeignKeys(datastoreSnapshot)
+                .apply("Wait on phase one", Wait.on(blocker)));
+    // Set up the pipeline to write entity kinds from PHASE_TWO_ORDERED to SQL. This phase won't
+    // start until all cleansed DomainBases have been written (started by line above).
+    scheduleOnePhaseWrites(
+        datastoreSnapshot, PHASE_TWO_ORDERED, Optional.of(blocker), "DomainBaseNoFkeys");
+  }
+
+  private PCollection<VersionedEntity> removeDomainBaseForeignKeys(
+      PCollectionTuple datastoreSnapshot) {
+    PCollection<VersionedEntity> domainBases =
+        datastoreSnapshot.get(Transforms.createTagForKind("DomainBase"));
+    return domainBases.apply(
+        "Remove circular foreign keys from DomainBase",
+        ParDo.of(new RemoveDomainBaseForeignKeys()));
+  }
+
+  /**
+   * Sets up the pipeline to write entities in {@code entityClasses} to SQL. Entities are written
+   * one kind at a time based on each kind's position in {@code entityClasses}. Concurrency exists
+   * within each kind.
+   *
+   * @param datastoreSnapshot the Datastore snapshot of all data to be migrated to SQL
+   * @param entityClasses the entity types in write order
+   * @param blockingPCollection the pipeline stage that blocks this phase
+   * @param blockingTag description of the stage (if exists) that blocks this phase. Needed for
+   *     generating unique transform ids
+   * @return the output {@code PCollection} from the writing of the last entity kind. Other parts of
+   *     the pipeline can {@link Wait} on this object
+   */
+  private PCollection<Void> scheduleOnePhaseWrites(
+      PCollectionTuple datastoreSnapshot,
+      Collection<Class<?>> entityClasses,
+      Optional<PCollection<Void>> blockingPCollection,
+      String blockingTag) {
+    checkArgument(!entityClasses.isEmpty(), "Each phase must have at least one kind.");
+    ImmutableList<TupleTag<VersionedEntity>> tags =
+        toKindStrings(entityClasses).stream()
+            .map(Transforms::createTagForKind)
+            .collect(ImmutableList.toImmutableList());
+
+    PCollection<Void> prev = blockingPCollection.orElse(null);
+    String prevTag = blockingTag;
+    for (TupleTag<VersionedEntity> tag : tags) {
+      PCollection<VersionedEntity> curr = datastoreSnapshot.get(tag);
+      if (prev != null) {
+        curr = curr.apply("Wait on " + prevTag, Wait.on(prev));
+      }
+      prev = writeToSql(tag.getId(), curr);
+      prevTag = tag.getId();
+    }
+    return prev;
+  }
+
+  private PCollection<Void> writeToSql(String transformId, PCollection<VersionedEntity> data) {
+    String credentialFileUrl =
+        options.getSqlCredentialUrlOverride() != null
+            ? options.getSqlCredentialUrlOverride()
+            : BackupPaths.getCloudSQLCredentialFilePatterns(options.getEnvironment()).get(0);
+
+    return data.apply(
+        "Write to sql: " + transformId,
+        Transforms.writeToSql(
+            transformId,
+            options.getMaxConcurrentSqlWriters(),
+            options.getSqlWriteBatchSize(),
+            new JpaSupplierFactory(credentialFileUrl, options.getCloudKmsProjectId(), jpaGetter)));
+  }
+
+  private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
+    try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+      return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
+    }
+  }
+
+  public static void main(String[] args) {
+    InitSqlPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(InitSqlPipelineOptions.class);
+    new InitSqlPipeline(options).run();
+  }
+}

core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java

@@ -0,0 +1,91 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Validation;
+
+/** Pipeline options for {@link InitSqlPipeline} */
+public interface InitSqlPipelineOptions extends GcpOptions {
+
+  @Description(
+      "Overrides the URL to the SQL credential file. " + "Required if environment is not provided.")
+  @Nullable
+  String getSqlCredentialUrlOverride();
+
+  void setSqlCredentialUrlOverride(String credentialUrlOverride);
+
+  @Description("The root directory of the export to load.")
+  String getDatastoreExportDir();
+
+  void setDatastoreExportDir(String datastoreExportDir);
+
+  @Description("The directory that contains all CommitLog files.")
+  String getCommitLogDir();
+
+  void setCommitLogDir(String commitLogDir);
+
+  @Description("The earliest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getCommitLogStartTimestamp();
+
+  void setCommitLogStartTimestamp(String commitLogStartTimestamp);
+
+  @Description("The latest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getCommitLogEndTimestamp();
+
+  void setCommitLogEndTimestamp(String commitLogEndTimestamp);
+
+  @Description(
+      "The deployed environment, alpha, crash, sandbox, or production. "
+          + "Not required only if sqlCredentialUrlOverride is provided.")
+  @Nullable
+  String getEnvironment();
+
+  void setEnvironment(String environment);
+
+  @Description(
+      "The GCP project that contains the keyring used for decrypting the " + "SQL credential file.")
+  @Nullable
+  String getCloudKmsProjectId();
+
+  void setCloudKmsProjectId(String cloudKmsProjectId);
+
+  @Description(
+      "The maximum JDBC connection pool size on a VM. "
+          + "This value should be equal to or greater than the number of cores on the VM.")
+  @Default.Integer(4)
+  int getJdbcMaxPoolSize();
+
+  void setJdbcMaxPoolSize(int jdbcMaxPoolSize);
+
+  @Description(
+      "A hint to the pipeline runner of the maximum number of concurrent SQL writers to create. "
+          + "Note that multiple writers may run on the same VM and share the connection pool.")
+  @Default.Integer(4)
+  int getMaxConcurrentSqlWriters();
+
+  void setMaxConcurrentSqlWriters(int maxConcurrentSqlWriters);
+
+  @Description("The number of entities to be written to the SQL database in one transaction.")
+  @Default.Integer(20)
+  int getSqlWriteBatchSize();
+
+  void setSqlWriteBatchSize(int sqlWriteBatchSize);
+}

core/src/main/java/google/registry/beam/initsql/JpaSupplierFactory.java

@@ -0,0 +1,48 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import google.registry.beam.initsql.BeamJpaModule.JpaTransactionManagerComponent;
+import google.registry.beam.initsql.Transforms.SerializableSupplier;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+
+public class JpaSupplierFactory implements SerializableSupplier<JpaTransactionManager> {
+
+  private static final long serialVersionUID = 1L;
+
+  private final String credentialFileUrl;
+  @Nullable private final String cloudKmsProjectId;
+  private final SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager>
+      jpaGetter;
+
+  public JpaSupplierFactory(
+      String credentialFileUrl,
+      @Nullable String cloudKmsProjectId,
+      SerializableFunction<JpaTransactionManagerComponent, JpaTransactionManager> jpaGetter) {
+    this.credentialFileUrl = credentialFileUrl;
+    this.cloudKmsProjectId = cloudKmsProjectId;
+    this.jpaGetter = jpaGetter;
+  }
+
+  @Override
+  public JpaTransactionManager get() {
+    return jpaGetter.apply(
+        DaggerBeamJpaModule_JpaTransactionManagerComponent.builder()
+            .beamJpaModule(new BeamJpaModule(credentialFileUrl, cloudKmsProjectId))
+            .build());
+  }
+}

core/src/main/java/google/registry/beam/initsql/README.md

@@ -1,3 +1,17 @@
 ## Summary
 
-This package contains a BEAM pipeline that populates a Cloud SQL database from a Datastore backup.
+This package contains a BEAM pipeline that populates a Cloud SQL database from a
+Datastore backup. The pipeline uses an unsynchronized Datastore export and
+overlapping CommitLogs generated by the Nomulus server to recreate a consistent
+Datastore snapshot, and writes the data to a Cloud SQL instance.
+
+## Pipeline Visualization
+
+The golden flow graph of the InitSqlPipeline is saved both as a text-base
+[DOT file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.dot)
+and a
+[.png file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.png).
+A test compares the flow graph of the current pipeline with the golden graph,
+and will fail if changes are detected. When this happens, run the Gradle task
+':core:updateInitSqlPipelineGraph' to update the golden files and review the
+changes.

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -17,26 +17,42 @@ package google.registry.beam.initsql;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Throwables.throwIfUnchecked;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
+import static google.registry.persistence.JpaRetries.isFailedTxnRetriable;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.setJpaTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static java.util.Comparator.comparing;
+import static org.apache.beam.sdk.values.TypeDescriptors.integers;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
 
 import avro.shaded.com.google.common.collect.Iterators;
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Streams;
+import google.registry.backup.AppEngineEnvironment;
 import google.registry.backup.CommitLogImports;
 import google.registry.backup.VersionedEntity;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.ofy.Ofy;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.tools.LevelDbLogReader;
+import google.registry.util.SystemSleeper;
+import java.io.Serializable;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Supplier;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.Compression;
 import org.apache.beam.sdk.io.FileIO;
@@ -47,6 +63,7 @@ import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.GroupIntoBatches;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -60,6 +77,7 @@ import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 
 /**
  * {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export
@@ -209,7 +227,7 @@ public final class Transforms {
     return new PTransform<PCollection<String>, PCollection<Metadata>>() {
       @Override
       public PCollection<Metadata> expand(PCollection<String> input) {
-        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW));
       }
     };
   }
@@ -245,6 +263,42 @@ public final class Transforms {
                     .iterator()));
   }
 
+  /**
+   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
+   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
+   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   *
+   * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
+   * job, we will not add features unless proven necessary.
+   *
+   * @param transformId a unique ID for an instance of the returned transform
+   * @param maxWriters the max number of concurrent writes to SQL, which also determines the max
+   *     number of connection pools created
+   * @param batchSize the number of entities to write in each operation
+   * @param jpaSupplier supplier of a {@link JpaTransactionManager}
+   */
+  public static PTransform<PCollection<VersionedEntity>, PCollection<Void>> writeToSql(
+      String transformId,
+      int maxWriters,
+      int batchSize,
+      SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+    return new PTransform<PCollection<VersionedEntity>, PCollection<Void>>() {
+      @Override
+      public PCollection<Void> expand(PCollection<VersionedEntity> input) {
+        return input
+            .apply(
+                "Shard data for " + transformId,
+                MapElements.into(kvs(integers(), TypeDescriptor.of(VersionedEntity.class)))
+                    .via(ve -> KV.of(ThreadLocalRandom.current().nextInt(maxWriters), ve)))
+            .apply("Batch output by shard " + transformId, GroupIntoBatches.ofSize(batchSize))
+            .apply("Write in batch for " + transformId, ParDo.of(new SqlBatchWriter(jpaSupplier)));
+      }
+    };
+  }
+
+  /** Interface for serializable {@link Supplier suppliers}. */
+  public interface SerializableSupplier<T> extends Supplier<T>, Serializable {}
+
   /**
    * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
    * given {@link Iterable}.
@@ -322,4 +376,116 @@ public final class Transforms {
       }
     }
   }
+
+  /**
+   * Writes a batch of entities to a SQL database.
+   *
+   * <p>Note that an arbitrary number of instances of this class may be created and freed in
+   * arbitrary order in a single JVM. Due to the tech debt that forced us to use a static variable
+   * to hold the {@code JpaTransactionManager} instance, we must ensure that JpaTransactionManager
+   * is not changed or torn down while being used by some instance.
+   */
+  private static class SqlBatchWriter extends DoFn<KV<Integer, Iterable<VersionedEntity>>, Void> {
+
+    private static int instanceCount = 0;
+    private static JpaTransactionManager originalJpa;
+
+    private final SerializableSupplier<JpaTransactionManager> jpaSupplier;
+
+    private transient Ofy ofy;
+    private transient SystemSleeper sleeper;
+
+    SqlBatchWriter(SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+      this.jpaSupplier = jpaSupplier;
+    }
+
+    @Setup
+    public void setup() {
+      sleeper = new SystemSleeper();
+
+      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+        ObjectifyService.initOfy();
+        ofy = ObjectifyService.ofy();
+      }
+
+      synchronized (SqlBatchWriter.class) {
+        if (instanceCount == 0) {
+          originalJpa = jpaTm();
+          setJpaTm(jpaSupplier);
+        }
+        instanceCount++;
+      }
+    }
+
+    @Teardown
+    public void teardown() {
+      synchronized (SqlBatchWriter.class) {
+        instanceCount--;
+        if (instanceCount == 0) {
+          jpaTm().teardown();
+          setJpaTm(() -> originalJpa);
+        }
+      }
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<Integer, Iterable<VersionedEntity>> kv) {
+      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+        ImmutableList<Object> ofyEntities =
+            Streams.stream(kv.getValue())
+                .map(VersionedEntity::getEntity)
+                .map(Optional::get)
+                .map(ofy::toPojo)
+                .collect(ImmutableList.toImmutableList());
+        retry(() -> jpaTm().transact(() -> jpaTm().saveNewOrUpdateAll(ofyEntities)));
+      }
+    }
+
+    // TODO(b/160632289): Enhance Retrier and use it here.
+    private void retry(Runnable runnable) {
+      int maxAttempts = 5;
+      int initialDelayMillis = 100;
+      double jitterRatio = 0.2;
+
+      for (int attempt = 0; attempt < maxAttempts; attempt++) {
+        try {
+          runnable.run();
+          return;
+        } catch (Throwable throwable) {
+          if (!isFailedTxnRetriable(throwable)) {
+            throwIfUnchecked(throwable);
+            throw new RuntimeException(throwable);
+          }
+          int sleepMillis = (1 << attempt) * initialDelayMillis;
+          int jitter =
+              ThreadLocalRandom.current().nextInt((int) (sleepMillis * jitterRatio))
+                  - (int) (sleepMillis * jitterRatio / 2);
+          sleeper.sleepUninterruptibly(Duration.millis(sleepMillis + jitter));
+        }
+      }
+    }
+  }
+
+  /**
+   * Removes BillingEvents, {@link google.registry.model.poll.PollMessage PollMessages} and {@link
+   * google.registry.model.host.HostResource} from a {@link DomainBase}. These are circular foreign
+   * key constraints that prevent migration of {@code DomainBase} to SQL databases.
+   *
+   * <p>See {@link InitSqlPipeline} for more information.
+   */
+  static class RemoveDomainBaseForeignKeys extends DoFn<VersionedEntity, VersionedEntity> {
+
+    @ProcessElement
+    public void processElement(
+        @Element VersionedEntity domainBase, OutputReceiver<VersionedEntity> out) {
+      checkArgument(
+          domainBase.getEntity().isPresent(), "Unexpected delete entity %s", domainBase.key());
+      Entity outputEntity =
+          DomainBaseUtil.removeBillingAndPollAndHosts(domainBase.getEntity().get());
+      out.output(
+          VersionedEntity.from(
+              domainBase.commitTimeMills(),
+              EntityTranslator.convertToPb(outputEntity).toByteArray()));
+    }
+  }
 }

core/src/main/java/google/registry/beam/spec11/SafeBrowsingTransforms.java

@@ -141,7 +141,7 @@ public class SafeBrowsingTransforms {
     @ProcessElement
     public void processElement(ProcessContext context) {
       Subdomain subdomain = context.element();
-      subdomainBuffer.put(subdomain.fullyQualifiedDomainName(), subdomain);
+      subdomainBuffer.put(subdomain.domainName(), subdomain);
       if (subdomainBuffer.size() >= BATCH_SIZE) {
         ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
         results.forEach(context::output);
@@ -239,7 +239,7 @@ public class SafeBrowsingTransforms {
             String url = match.getJSONObject("threat").getString("url");
             Subdomain subdomain = subdomainBuffer.get(url);
             resultBuilder.add(
-                KV.of(subdomain, ThreatMatch.create(match, subdomain.fullyQualifiedDomainName())));
+                KV.of(subdomain, ThreatMatch.create(match, subdomain.domainName())));
           }
         }
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -77,7 +77,7 @@ public class Spec11Pipeline implements Serializable {
   public static final String REGISTRAR_EMAIL_FIELD = "registrarEmailAddress";
   /** The JSON object field into which we put the registrar's name for Spec11 reports. */
   public static final String REGISTRAR_CLIENT_ID_FIELD = "registrarClientId";
-  /** The JSON object field we put the threat match array for Spec11 reports. */
+  /** The JSON object field into which we put the threat match array for Spec11 reports. */
   public static final String THREAT_MATCHES_FIELD = "threatMatches";
 
   private final String projectId;
@@ -176,9 +176,11 @@ public class Spec11Pipeline implements Serializable {
       PCollection<Subdomain> domains,
       EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
       ValueProvider<String> dateProvider) {
-    PCollection<KV<Subdomain, ThreatMatch>> subdomains =
+
+    /* Store ThreatMatch objects in JSON. */
+    PCollection<KV<Subdomain, ThreatMatch>> subdomainsJson =
         domains.apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn));
-    subdomains
+    subdomainsJson
         .apply(
             "Map registrar client ID to email/ThreatMatch pair",
             MapElements.into(
@@ -187,7 +189,7 @@ public class Spec11Pipeline implements Serializable {
                 .via(
                     (KV<Subdomain, ThreatMatch> kv) ->
                         KV.of(
-                            kv.getKey().registrarClientId(),
+                            kv.getKey().registrarId(),
                             EmailAndThreatMatch.create(
                                 kv.getKey().registrarEmailAddress(), kv.getValue()))))
         .apply("Group by registrar client ID", GroupByKey.create())

core/src/main/java/google/registry/beam/spec11/Subdomain.java

@@ -36,12 +36,14 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 public abstract class Subdomain implements Serializable {
 
   private static final ImmutableList<String> FIELD_NAMES =
-      ImmutableList.of("fullyQualifiedDomainName", "registrarClientId", "registrarEmailAddress");
+      ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress");
 
   /** Returns the fully qualified domain name. */
-  abstract String fullyQualifiedDomainName();
-  /** Returns the client ID of the associated registrar for this domain. */
-  abstract String registrarClientId();
+  abstract String domainName();
+  /** Returns the domain repo ID (the primary key of the domain table). */
+  abstract String domainRepoId();
+  /** Returns the registrar ID of the associated registrar for this domain. */
+  abstract String registrarId();
   /** Returns the email address of the registrar associated with this domain. */
   abstract String registrarEmailAddress();
 
@@ -56,8 +58,9 @@ public abstract class Subdomain implements Serializable {
     checkFieldsNotNull(FIELD_NAMES, schemaAndRecord);
     GenericRecord record = schemaAndRecord.getRecord();
     return create(
-        extractField(record, "fullyQualifiedDomainName"),
-        extractField(record, "registrarClientId"),
+        extractField(record, "domainName"),
+        extractField(record, "domainRepoId"),
+        extractField(record, "registrarId"),
         extractField(record, "registrarEmailAddress"));
   }
 
@@ -69,9 +72,11 @@ public abstract class Subdomain implements Serializable {
    */
   @VisibleForTesting
   static Subdomain create(
-      String fullyQualifiedDomainName, String registrarClientId, String registrarEmailAddress) {
+      String domainName,
+      String domainRepoId,
+      String registrarId,
+      String registrarEmailAddress) {
     return new AutoValue_Subdomain(
-        fullyQualifiedDomainName, registrarClientId, registrarEmailAddress);
+        domainName, domainRepoId, registrarId, registrarEmailAddress);
   }
 }
-

core/src/main/java/google/registry/beam/spec11/sql/subdomains.sql

@@ -19,11 +19,13 @@
   -- email address.
 
 SELECT
-  domain.fullyQualifiedDomainName AS fullyQualifiedDomainName,
-  registrar.clientId AS registrarClientId,
+  domain.fullyQualifiedDomainName AS domainName,
+  domain.__key__.name AS domainRepoId,
+  registrar.clientId AS registrarId,
   COALESCE(registrar.emailAddress, '') AS registrarEmailAddress
 FROM ( (
     SELECT
+      __key__,
       fullyQualifiedDomainName,
       currentSponsorClientId,
       creationTime

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -1527,6 +1527,21 @@ public final class RegistryConfig {
     return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
   }
 
+  /**
+   * Returns whether to replicate cloud SQL transactions to datastore.
+   *
+   * <p>If true, all cloud SQL transactions will be persisted as TransactionEntity objects in the
+   * Transaction table and replayed against datastore in a cron job.
+   */
+  public static boolean getCloudSqlReplicateTransactions() {
+    return CONFIG_SETTINGS.get().cloudSql.replicateTransactions;
+  }
+
+  @VisibleForTesting
+  public static void overrideCloudSqlReplicateTransactions(boolean replicateTransactions) {
+    CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
+  }
+
   /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
   public static String getContactAndHostRoidSuffix() {
     return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -122,6 +122,7 @@ public class RegistryConfigSettings {
     public String jdbcUrl;
     public String username;
     public String instanceConnectionName;
+    public boolean replicateTransactions;
   }
 
   /** Configuration for Apache Beam (Cloud Dataflow). */

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -230,6 +230,9 @@ cloudSql:
   username: username
   # This name is used by Cloud SQL when connecting to the database.
   instanceConnectionName: project-id:region:instance-id
+  # Set this to true to replicate cloud SQL transactions to datastore in the
+  # background.
+  replicateTransactions: false
 
 cloudDns:
   # Set both properties to null in Production.

core/src/main/java/google/registry/documentation/FlowDocumentation.java

@@ -1,275 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.documentation;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.MoreCollectors.onlyElement;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Multimaps;
-import com.sun.javadoc.AnnotationDesc;
-import com.sun.javadoc.ClassDoc;
-import com.sun.javadoc.FieldDoc;
-import com.sun.javadoc.SeeTag;
-import com.sun.javadoc.Tag;
-import google.registry.model.eppoutput.Result.Code;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.TreeMap;
-import java.util.stream.Stream;
-import javax.annotation.Nullable;
-
-/**
- * Class to represent documentation information for a single EPP flow.
- *
- * <p>The static method getFlowDocs() on this class returns a list of FlowDocumentation
- * instances corresponding to the leaf flows in the flows package, constructing the instances
- * from class information returned from the javadoc system.  Each instance has methods for
- * retrieving relevant information about the flow, such as a description, error conditions, etc.
- */
-public class FlowDocumentation {
-
-  /** Constants for names of various relevant packages and classes. */
-  static final String FLOW_PACKAGE_NAME = "google.registry.flows";
-  static final String BASE_FLOW_CLASS_NAME = FLOW_PACKAGE_NAME + ".Flow";
-  static final String EXCEPTION_CLASS_NAME = FLOW_PACKAGE_NAME + ".EppException";
-  static final String CODE_ANNOTATION_NAME = EXCEPTION_CLASS_NAME + ".EppResultCode";
-
-  /** Name of the class for this flow. */
-  private final String name;
-
-  /** Fully qualified name of the class for this flow. */
-  private final String qualifiedName;
-
-  /** Name of the package in which this flow resides. */
-  private final String packageName;
-
-  /** Class docs for the flow. */
-  private final String classDocs;
-
-  /** Javadoc-tagged error conditions for this flow in list form. */
-  private final List<ErrorCase> errors;
-
-  /** Javadoc-tagged error conditions for this flow, organized by underlying error code. */
-  private final ListMultimap<Long, ErrorCase> errorsByCode;
-
-  /**
-   * Creates a FlowDocumentation for this flow class using data from javadoc tags.  Not public
-   * because clients should get FlowDocumentation objects via the DocumentationGenerator class.
-   */
-  protected FlowDocumentation(ClassDoc flowDoc) {
-    name = flowDoc.name();
-    qualifiedName = flowDoc.qualifiedName();
-    packageName = flowDoc.containingPackage().name();
-    classDocs = flowDoc.commentText();
-    errors = new ArrayList<>();
-    // Store error codes in sorted order, and leave reasons in insert order.
-    errorsByCode =
-        Multimaps.newListMultimap(new TreeMap<Long, Collection<ErrorCase>>(), ArrayList::new);
-    parseTags(flowDoc);
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  public String getQualifiedName() {
-    return qualifiedName;
-  }
-
-  public String getPackageName() {
-    return packageName;
-  }
-
-  public String getClassDocs() {
-    return classDocs;
-  }
-
-  public ImmutableList<ErrorCase> getErrors() {
-    return ImmutableList.copyOf(errors);
-  }
-
-  public ImmutableMultimap<Long, ErrorCase> getErrorsByCode() {
-    return ImmutableMultimap.copyOf(errorsByCode);
-  }
-
-  /** Iterates through javadoc tags on the underlying class and calls specific parsing methods. */
-  private void parseTags(ClassDoc flowDoc) {
-    for (Tag tag : flowDoc.tags()) {
-      // Everything else is not a relevant tag.
-      if ("@error".equals(tag.name())) {
-        parseErrorTag(tag);
-      }
-    }
-  }
-
-  /** Exception to throw when an @error tag cannot be parsed correctly. */
-  private static class BadErrorTagFormatException extends IllegalStateException {
-    /** Makes a message to use as a prefix for the reason passed up to the superclass. */
-    private static String makeMessage(String reason, Tag tag) {
-      return String.format("Bad @error tag format at %s - %s", tag.position(), reason);
-    }
-
-    private BadErrorTagFormatException(String reason, Tag tag) {
-      super(makeMessage(reason, tag));
-    }
-
-    private BadErrorTagFormatException(String reason, Tag tag, Exception cause) {
-      super(makeMessage(reason, tag), cause);
-    }
-  }
-
-  /** Parses a javadoc tag corresponding to an error case and updates the error mapping. */
-  private void parseErrorTag(Tag tag) {
-    // Parse the @error tag text to find the @link inline tag.
-    SeeTag linkedTag;
-    try {
-      linkedTag =
-          Stream.of(tag.inlineTags())
-              .filter(SeeTag.class::isInstance)
-              .map(SeeTag.class::cast)
-              .collect(onlyElement());
-    } catch (NoSuchElementException | IllegalArgumentException e) {
-      throw new BadErrorTagFormatException(
-          String.format("expected one @link tag in tag text but found %s: %s",
-              (e instanceof NoSuchElementException ? "none" : "multiple"),
-              tag.text()),
-          tag, e);
-    }
-    // Check to see if the @link tag references a valid class.
-    ClassDoc exceptionRef = linkedTag.referencedClass();
-    if (exceptionRef == null) {
-      throw new BadErrorTagFormatException(
-          "could not resolve class from @link tag text: " + linkedTag.text(),
-          tag);
-    }
-    // Try to convert the referenced class into an ErrorCase; fail if it's not an EppException.
-    ErrorCase error;
-    try {
-      error = new ErrorCase(exceptionRef);
-    } catch (IllegalStateException | IllegalArgumentException e) {
-      throw new BadErrorTagFormatException(
-          "class referenced in @link is not a valid EppException: " + exceptionRef.qualifiedName(),
-          tag, e);
-    }
-    // Success; store this as a parsed error case.
-    errors.add(error);
-    errorsByCode.put(error.getCode(), error);
-  }
-
-  /**
-   * Represents an error case for a flow, with a reason for the error and the EPP error code.
-   *
-   * <p>This class is an immutable wrapper for the name of an EppException subclass that gets
-   * thrown to indicate an error condition.  It overrides equals() and hashCode() so that
-   * instances of this class can be used in collections in the normal fashion.
-   */
-  public static class ErrorCase {
-
-    /** The non-qualified name of the exception class. */
-    private final String name;
-
-    /** The fully-qualified name of the exception class. */
-    private final String className;
-
-    /** The reason this error was thrown, normally documented on the low-level exception class. */
-    private final String reason;
-
-    /** The EPP error code value corresponding to this error condition. */
-    private final long errorCode;
-
-    /** Constructs an ErrorCase from the corresponding class for a low-level flow exception. */
-    protected ErrorCase(ClassDoc exceptionDoc) {
-      name = exceptionDoc.name();
-      className = exceptionDoc.qualifiedName();
-      // The javadoc comment on the class explains the reason for the error condition.
-      reason = exceptionDoc.commentText();
-      ClassDoc highLevelExceptionDoc = getHighLevelExceptionFrom(exceptionDoc);
-      errorCode = extractErrorCode(highLevelExceptionDoc);
-      checkArgument(!exceptionDoc.isAbstract(),
-          "Cannot use an abstract subclass of EppException as an error case");
-    }
-
-    public String getName() {
-      return name;
-    }
-
-    protected String getClassName() {
-      return className;
-    }
-
-    public String getReason() {
-      return reason;
-    }
-
-    public long getCode() {
-      return errorCode;
-    }
-
-    /** Returns the direct subclass of EppException that this class is a subclass of (or is). */
-    private ClassDoc getHighLevelExceptionFrom(ClassDoc exceptionDoc) {
-      // While we're not yet at the root, move up the class hierarchy looking for EppException.
-      while (exceptionDoc.superclass() != null) {
-        if (exceptionDoc.superclass().qualifiedTypeName().equals(EXCEPTION_CLASS_NAME)) {
-          return exceptionDoc;
-        }
-        exceptionDoc = exceptionDoc.superclass();
-      }
-      // Failure; we reached the root without finding a subclass of EppException.
-      throw new IllegalArgumentException(
-          String.format("Class referenced is not a subclass of %s", EXCEPTION_CLASS_NAME));
-    }
-
-    /** Returns the corresponding EPP error code for an annotated subclass of EppException. */
-    private long extractErrorCode(ClassDoc exceptionDoc) {
-      try {
-        // We're looking for a specific annotation by name that should appear only once.
-        AnnotationDesc errorCodeAnnotation =
-            Arrays.stream(exceptionDoc.annotations())
-                .filter(
-                    anno -> anno.annotationType().qualifiedTypeName().equals(CODE_ANNOTATION_NAME))
-                .findFirst()
-                .get();
-        // The annotation should have one element whose value converts to an EppResult.Code.
-        AnnotationDesc.ElementValuePair pair = errorCodeAnnotation.elementValues()[0];
-        String enumConstant = ((FieldDoc) pair.value().value()).name();
-        return Code.valueOf(enumConstant).code;
-      } catch (IllegalStateException e) {
-        throw new IllegalStateException(
-            "No error code annotation found on exception " + exceptionDoc.name(), e);
-      } catch (ArrayIndexOutOfBoundsException | ClassCastException | IllegalArgumentException e) {
-        throw new IllegalStateException("Bad annotation on exception " + exceptionDoc.name(), e);
-      }
-    }
-
-    @Override
-    public boolean equals(@Nullable Object object) {
-      // The className field canonically identifies the EppException wrapped by this class, and
-      // all other instance state is derived from that exception, so we only check className.
-      return object instanceof ErrorCase && this.className.equals(((ErrorCase) object).className);
-    }
-
-    @Override
-    public int hashCode() {
-      // See note for equals() - only className is needed for comparisons.
-      return className.hashCode();
-    }
-  }
-}

core/src/main/java/google/registry/documentation/JavadocWrapper.java

@@ -1,159 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.documentation;
-
-import static google.registry.util.BuildPathUtils.getProjectRoot;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.flogger.FluentLogger;
-import com.google.common.io.CharStreams;
-import com.sun.javadoc.RootDoc;
-import com.sun.tools.javac.file.JavacFileManager;
-import com.sun.tools.javac.util.Context;
-import com.sun.tools.javac.util.ListBuffer;
-import com.sun.tools.javadoc.JavadocTool;
-import com.sun.tools.javadoc.Messager;
-import com.sun.tools.javadoc.ModifierFilter;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Locale;
-import javax.annotation.Nullable;
-import javax.tools.StandardLocation;
-
-/**
- * Wrapper class to simplify calls to the javadoc system and hide internal details.  An instance
- * represents a set of parameters for calling out to javadoc; these parameters can be set via
- * the appropriate methods, and determine what files and packages javadoc will process.  The
- * actual running of javadoc occurs when calling getRootDoc() to retrieve a javadoc RootDoc.
- */
-public final class JavadocWrapper {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
-  /** Shows any member visible at at least the default (package) level. */
-  private static final long VISIBILITY_MASK =
-      Modifier.PUBLIC | Modifier.PROTECTED | ModifierFilter.PACKAGE;
-
-  /** Root directory for source files. If null, will use the current directory. */
-  private static final String SOURCE_PATH = getProjectRoot().resolve("core/src/main/java")
-      .toString();
-  /** Specific source files to generate documentation for. */
-  private static final ImmutableSet<String> SOURCE_FILE_NAMES = ImmutableSet.of();
-
-  /** Specific packages to generate documentation for. */
-  private static final ImmutableSet<String> SOURCE_PACKAGE_NAMES =
-      ImmutableSet.of(FlowDocumentation.FLOW_PACKAGE_NAME);
-
-  /** Whether or not the Javadoc tool should eschew excessive log output. */
-  private static final boolean QUIET = true;
-
-  /**
-   * Obtains a Javadoc {@link RootDoc} object containing raw Javadoc documentation.
-   * Wraps a call to the static method createRootDoc() and passes in instance-specific settings.
-   */
-  public static RootDoc getRootDoc() throws IOException {
-    logger.atInfo().log("Starting Javadoc tool");
-    File sourceFilePath = new File(SOURCE_PATH);
-    logger.atInfo().log("Using source directory: %s", sourceFilePath.getAbsolutePath());
-    try {
-      return createRootDoc(
-          SOURCE_PATH,
-          SOURCE_PACKAGE_NAMES,
-          SOURCE_FILE_NAMES,
-          VISIBILITY_MASK,
-          QUIET);
-    } finally {
-      logger.atInfo().log("Javadoc tool finished");
-    }
-  }
-
-  /**
-   * Obtains a Javadoc root document object for the specified source path and package/Java names.
-   * If the source path is null, then the working directory is assumed as the source path.
-   *
-   * <p>If a list of package names is provided, then Javadoc will run on these packages and all
-   * their subpackages, based out of the specified source path.
-   *
-   * <p>If a list of file names is provided, then Javadoc will also run on these Java source files.
-   * The specified source path is not considered in this case.
-   *
-   * @see <a href="http://relation.to/12969.lace">Testing Java doclets</a>
-   * @see <a href="http://www.docjar.com/docs/api/com/sun/tools/javadoc/JavadocTool.html">JavadocTool</a>
-   */
-  private static RootDoc createRootDoc(
-      @Nullable String sourcePath,
-      Collection<String> packageNames,
-      Collection<String> fileNames,
-      long visibilityMask,
-      boolean quiet) throws IOException {
-    // Create a context to hold settings for Javadoc.
-    Context context = new Context();
-
-    // Redirect Javadoc stdout/stderr to null writers, since otherwise the Java compiler
-    // issues lots of errors for classes that are imported and visible to blaze but not
-    // visible locally to the compiler.
-    // TODO(b/19124943): Find a way to ignore those errors so we can show real ones?
-    Messager.preRegister(
-        context,
-        JavadocWrapper.class.getName(),
-        new PrintWriter(CharStreams.nullWriter()),     // For errors.
-        new PrintWriter(CharStreams.nullWriter()),     // For warnings.
-        new PrintWriter(CharStreams.nullWriter()));    // For notices.
-
-    // Set source path option for Javadoc.
-    try (JavacFileManager fileManager = new JavacFileManager(context, true, UTF_8)) {
-      List<File> sourcePathFiles = new ArrayList<>();
-      if (sourcePath != null) {
-        for (String sourcePathEntry : Splitter.on(':').split(sourcePath)) {
-          sourcePathFiles.add(new File(sourcePathEntry));
-        }
-      }
-      fileManager.setLocation(StandardLocation.SOURCE_PATH, sourcePathFiles);
-
-      // Create an instance of Javadoc.
-      JavadocTool javadocTool = JavadocTool.make0(context);
-
-      // Convert the package and file lists to a format Javadoc can understand.
-      ListBuffer<String> subPackages = new ListBuffer<>();
-      subPackages.addAll(packageNames);
-      ListBuffer<String> javaNames = new ListBuffer<>();
-      javaNames.addAll(fileNames);
-
-      // Invoke Javadoc and ask it for a RootDoc containing the specified packages.
-      return javadocTool.getRootDocImpl(
-          Locale.US.toString(),                    // Javadoc comment locale
-          UTF_8.name(),                            // Source character encoding
-          new ModifierFilter(visibilityMask),      // Element visibility filter
-          javaNames.toList(),                      // Included Java file names
-          com.sun.tools.javac.util.List.nil(),     // Doclet options
-          com.sun.tools.javac.util.List.nil(),     // Source files
-          false,                                   // Don't use BreakIterator
-          subPackages.toList(),                    // Included sub-package names
-          com.sun.tools.javac.util.List.nil(),     // Excluded package names
-          false,                                   // Read source files, not classes
-          false,                                   // Don't run legacy doclet
-          quiet);                                  // If asked, run Javadoc quietly
-    }
-  }
-
-  private JavadocWrapper() {}
-}

core/src/main/java/google/registry/documentation/generate_javadoc.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-# Copyright 2017 The Nomulus Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Generate javadoc for the project
-
-if (( $# != 3 )); then
-  echo "Usage: $0 JAVADOC ZIP OUT" 1>&2
-  exit 1
-fi
-
-JAVADOC_BINARY="$1"
-ZIP_BINARY="$2"
-TARGETFILE="$3"
-TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/generate_javadoc.XXXXXXXX")"
-PWDDIR="$(pwd)"
-
-"${JAVADOC_BINARY}" -d "${TMPDIR}" \
-  $(find java -name \*.java) \
-  -tag error:t:'EPP Errors' \
-  -subpackages google.registry \
-  -exclude google.registry.dns:google.registry.proxy:google.registry.monitoring.blackbox
-cd "${TMPDIR}"
-"${PWDDIR}/${ZIP_BINARY}" -rXoq "${PWDDIR}/${TARGETFILE}" .
-cd -
-rm -rf "${TMPDIR}"

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -367,6 +367,12 @@
     <url-pattern>/_dr/task/linkRdeHosts</url-pattern>
   </servlet-mapping>
 
+  <!-- Action to automatically re-lock a domain after unlocking it -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/relockDomain</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

core/src/main/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml

@@ -7,7 +7,7 @@
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <manual-scaling>
-    <instances>10</instances>
+    <instances>20</instances>
   </manual-scaling>
 
   <system-properties>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -18,7 +18,14 @@
       and streams it to cloud storage. When this job has finished successfully, it'll
       launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
     </description>
-    <schedule>every day 00:07</schedule>
+    <!--
+      This only needs to run once per day, but we launch additional jobs in case the
+      cursor is lagging behind, so it'll catch up to the current date eventually.
+
+      See <a href="../../../production/default/WEB-INF/cron.xml">production config</a> for an
+      explanation of job starting times.
+    -->
+    <schedule>every 12 hours from 00:07 to 12:07</schedule>
     <target>backend</target>
   </cron>
 

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -347,8 +347,8 @@ public class DomainCreateFlow implements TransactionalFlow {
             .setRepoId(repoId)
             .setIdnTableName(validateDomainNameWithIdnTables(domainName))
             .setRegistrationExpirationTime(registrationExpirationTime)
-            .setAutorenewBillingEvent(Key.create(autorenewBillingEvent))
-            .setAutorenewPollMessage(Key.create(autorenewPollMessage))
+            .setAutorenewBillingEvent(autorenewBillingEvent.createVKey())
+            .setAutorenewPollMessage(autorenewPollMessage.createVKey())
             .setLaunchNotice(hasClaimsNotice ? launchCreate.get().getNotice() : null)
             .setSmdId(signedMarkId)
             .setDsData(secDnsCreate.isPresent() ? secDnsCreate.get().getDsData() : null)

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -31,7 +31,6 @@ import static google.registry.model.ResourceTransferUtils.handlePendingTransferO
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.ADD_FIELDS;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.RENEW_FIELDS;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -209,7 +208,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
       PollMessage.OneTime deletePollMessage =
           createDeletePollMessage(existingDomain, historyEntry, deletionTime);
       entitiesToSave.add(deletePollMessage);
-      builder.setDeletePollMessage(Key.create(deletePollMessage));
+      builder.setDeletePollMessage(deletePollMessage.createVKey());
     }
 
     // Cancel any grace periods that were still active, and set the expiration time accordingly.
@@ -222,8 +221,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
         if (gracePeriod.getOneTimeBillingEvent() != null) {
           // Take the amount of amount of registration time being refunded off the expiration time.
           // This can be either add grace periods or renew grace periods.
-          BillingEvent.OneTime oneTime =
-              ofy().load().key(gracePeriod.getOneTimeBillingEvent()).now();
+          BillingEvent.OneTime oneTime = tm().load(gracePeriod.getOneTimeBillingEvent());
           newExpirationTime = newExpirationTime.minusYears(oneTime.getPeriodYears());
         } else if (gracePeriod.getRecurringBillingEvent() != null) {
           // Take 1 year off the registration if in the autorenew grace period (no need to load the
@@ -370,12 +368,12 @@ public final class DomainDeleteFlow implements TransactionalFlow {
   private Money getGracePeriodCost(GracePeriod gracePeriod, DateTime now) {
     if (gracePeriod.getType() == GracePeriodStatus.AUTO_RENEW) {
       DateTime autoRenewTime =
-          ofy().load().key(checkNotNull(gracePeriod.getRecurringBillingEvent())).now()
+          tm().load(checkNotNull(gracePeriod.getRecurringBillingEvent()))
               .getRecurrenceTimeOfYear()
-                  .getLastInstanceBeforeOrAt(now);
+              .getLastInstanceBeforeOrAt(now);
       return getDomainRenewCost(targetId, autoRenewTime, 1);
     }
-    return ofy().load().key(checkNotNull(gracePeriod.getOneTimeBillingEvent())).now().getCost();
+    return tm().load(checkNotNull(gracePeriod.getOneTimeBillingEvent())).getCost();
   }
 
   @Nullable

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -517,14 +517,14 @@ public class DomainFlowUtils {
    */
   public static void updateAutorenewRecurrenceEndTime(DomainBase domain, DateTime newEndTime) {
     Optional<PollMessage.Autorenew> autorenewPollMessage =
-        Optional.ofNullable(ofy().load().key(domain.getAutorenewPollMessage()).now());
+        tm().maybeLoad(domain.getAutorenewPollMessage());
 
     // Construct an updated autorenew poll message. If the autorenew poll message no longer exists,
     // create a new one at the same id. This can happen if a transfer was requested on a domain
     // where all autorenew poll messages had already been delivered (this would cause the poll
     // message to be deleted), and then subsequently the transfer was canceled, rejected, or deleted
     // (which would cause the poll message to be recreated here).
-    Key<PollMessage.Autorenew> existingAutorenewKey = domain.getAutorenewPollMessage();
+    Key<PollMessage.Autorenew> existingAutorenewKey = domain.getAutorenewPollMessage().getOfyKey();
     PollMessage.Autorenew updatedAutorenewPollMessage =
         autorenewPollMessage.isPresent()
             ? autorenewPollMessage.get().asBuilder().setAutorenewEndTime(newEndTime).build()
@@ -542,7 +542,7 @@ public class DomainFlowUtils {
       ofy().save().entity(updatedAutorenewPollMessage);
     }
 
-    Recurring recurring = ofy().load().key(domain.getAutorenewBillingEvent()).now();
+    Recurring recurring = tm().load(domain.getAutorenewBillingEvent());
     ofy().save().entity(recurring.asBuilder().setRecurrenceEndTime(newEndTime).build());
   }
 

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -33,7 +33,6 @@ import static google.registry.util.DateTimeUtils.leapSafeAddYears;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
 import google.registry.flows.EppException.ParameterValueRangeErrorException;
 import google.registry.flows.ExtensionManager;
@@ -181,8 +180,8 @@ public final class DomainRenewFlow implements TransactionalFlow {
             .setLastEppUpdateTime(now)
             .setLastEppUpdateClientId(clientId)
             .setRegistrationExpirationTime(newExpirationTime)
-            .setAutorenewBillingEvent(Key.create(newAutorenewEvent))
-            .setAutorenewPollMessage(Key.create(newAutorenewPollMessage))
+            .setAutorenewBillingEvent(newAutorenewEvent.createVKey())
+            .setAutorenewPollMessage(newAutorenewPollMessage.createVKey())
             .addGracePeriod(
                 GracePeriod.forBillingEvent(GracePeriodStatus.RENEW, explicitRenewEvent))
             .build();

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -26,7 +26,6 @@ import static google.registry.flows.domain.DomainFlowUtils.verifyNotReserved;
 import static google.registry.flows.domain.DomainFlowUtils.verifyPremiumNameIsNotBlocked;
 import static google.registry.flows.domain.DomainFlowUtils.verifyRegistrarIsActive;
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
@@ -174,8 +173,8 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
             existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId);
     updateForeignKeyIndexDeletionTime(newDomain);
     entitiesToSave.add(newDomain, historyEntry, autorenewEvent, autorenewPollMessage);
-    ofy().save().entities(entitiesToSave.build());
-    ofy().delete().key(existingDomain.getDeletePollMessage());
+    tm().saveNewOrUpdateAll(entitiesToSave.build());
+    tm().delete(existingDomain.getDeletePollMessage());
     dnsQueue.addDomainRefreshTask(existingDomain.getDomainName());
     return responseBuilder
         .setExtensions(createResponseExtensions(feesAndCredits, feeUpdate, isExpired))
@@ -232,8 +231,8 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
         .setStatusValues(null)
         .setGracePeriods(null)
         .setDeletePollMessage(null)
-        .setAutorenewBillingEvent(Key.create(autorenewEvent))
-        .setAutorenewPollMessage(Key.create(autorenewPollMessage))
+        .setAutorenewBillingEvent(autorenewEvent.createVKey())
+        .setAutorenewPollMessage(autorenewPollMessage.createVKey())
         .setLastEppUpdateTime(now)
         .setLastEppUpdateClientId(clientId)
         .build();

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -186,8 +186,8 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
                     .setTransferredRegistrationExpirationTime(newExpirationTime)
                     .build())
             .setRegistrationExpirationTime(newExpirationTime)
-            .setAutorenewBillingEvent(Key.create(autorenewEvent))
-            .setAutorenewPollMessage(Key.create(gainingClientAutorenewPollMessage))
+            .setAutorenewBillingEvent(autorenewEvent.createVKey())
+            .setAutorenewPollMessage(gainingClientAutorenewPollMessage.createVKey())
             // Remove all the old grace periods and add a new one for the transfer.
             .setGracePeriods(
                 billingEvent.isPresent()

core/src/main/java/google/registry/model/BackupGroupRoot.java

@@ -14,16 +14,21 @@
 
 package google.registry.model;
 
+import com.google.common.annotations.VisibleForTesting;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlTransient;
 
 /**
  * Base class for entities that are the root of a Registry 2.0 entity group that gets enrolled in
  * commit logs for backup purposes.
  *
- * <p>The commit log system needs to preserve the ordering of closely timed mutations to entities
- * in a single entity group. We require an {@link UpdateAutoTimestamp} field on the root of a group
- * so that we can enforce strictly increasing timestamps.
+ * <p>The commit log system needs to preserve the ordering of closely timed mutations to entities in
+ * a single entity group. We require an {@link UpdateAutoTimestamp} field on the root of a group so
+ * that we can enforce strictly increasing timestamps.
  */
+@MappedSuperclass
 public abstract class BackupGroupRoot extends ImmutableObject {
   /**
    * An automatically managed timestamp of when this object was last written to Datastore.
@@ -32,10 +37,14 @@ public abstract class BackupGroupRoot extends ImmutableObject {
    * that this is updated on every save, rather than only in response to an {@code <update>} command
    */
   @XmlTransient
+  // Prevents subclasses from unexpectedly accessing as property (e.g., HostResource), which would
+  // require an unnecessary non-private setter method.
+  @Access(AccessType.FIELD)
+  @VisibleForTesting
   UpdateAutoTimestamp updateTimestamp = UpdateAutoTimestamp.create(null);
 
   /** Get the {@link UpdateAutoTimestamp} for this entity. */
-  public final UpdateAutoTimestamp getUpdateAutoTimestamp() {
+  public UpdateAutoTimestamp getUpdateTimestamp() {
     return updateTimestamp;
   }
 }

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -155,7 +155,7 @@ public final class EppResourceUtils {
     // time for writes.
     return Optional.of(
         cloneProjectedAtTime(
-            resource, latestOf(now, resource.getUpdateAutoTimestamp().getTimestamp())));
+            resource, latestOf(now, resource.getUpdateTimestamp().getTimestamp())));
   }
 
   /**
@@ -298,7 +298,7 @@ public final class EppResourceUtils {
     // and returns it projected forward to exactly the desired timestamp, or null if the resource is
     // deleted at that timestamp.
     final Result<T> loadResult =
-        isAtOrAfter(timestamp, resource.getUpdateAutoTimestamp().getTimestamp())
+        isAtOrAfter(timestamp, resource.getUpdateTimestamp().getTimestamp())
             ? new ResultNow<>(resource)
             : loadMostRecentRevisionAtTime(resource, timestamp);
     return () -> {

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -630,9 +630,9 @@ public abstract class BillingEvent extends ImmutableObject
           .setParent(historyEntry);
       // Set the grace period's billing event using the appropriate Cancellation builder method.
       if (gracePeriod.getOneTimeBillingEvent() != null) {
-        builder.setOneTimeEventKey(VKey.from(gracePeriod.getOneTimeBillingEvent()));
+        builder.setOneTimeEventKey(gracePeriod.getOneTimeBillingEvent());
       } else if (gracePeriod.getRecurringBillingEvent() != null) {
-        builder.setRecurringEventKey(VKey.from(gracePeriod.getRecurringBillingEvent()));
+        builder.setRecurringEventKey(gracePeriod.getRecurringBillingEvent());
       }
       return builder.build();
     }

core/src/main/java/google/registry/model/contact/ContactBase.java

@@ -44,10 +44,11 @@ import org.joda.time.DateTime;
 /**
  * A persistable contact resource including mutable and non-mutable fields.
  *
+ * <p>This class deliberately does not include an {@link javax.persistence.Id} so that any
+ * foreign-keyed fields can refer to the proper parent entity's ID, whether we're storing this in
+ * the DB itself or as part of another entity
+ *
  * @see <a href="https://tools.ietf.org/html/rfc5733">RFC 5733</a>
- *     <p>This class deliberately does not include an {@link javax.persistence.Id} so that any
- *     foreign-keyed fields can refer to the proper parent entity's ID, whether we're storing this
- *     in the DB itself or as part of another entity
  */
 @MappedSuperclass
 @Embeddable
@@ -70,24 +71,24 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
   @IgnoreSave(IfNull.class)
   @Embedded
   @AttributeOverrides({
-      @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
-      @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
-      @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
-      @AttributeOverride(
-          name = "address.streetLine1",
-          column = @Column(name = "addr_local_street_line1")),
-      @AttributeOverride(
-          name = "address.streetLine2",
-          column = @Column(name = "addr_local_street_line2")),
-      @AttributeOverride(
-          name = "address.streetLine3",
-          column = @Column(name = "addr_local_street_line3")),
-      @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
-      @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
-      @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
-      @AttributeOverride(
-          name = "address.countryCode",
-          column = @Column(name = "addr_local_country_code"))
+    @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_local_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_local_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_local_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_local_country_code"))
   })
   PostalInfo localizedPostalInfo;
 
@@ -98,24 +99,24 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
   @IgnoreSave(IfNull.class)
   @Embedded
   @AttributeOverrides({
-      @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
-      @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
-      @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
-      @AttributeOverride(
-          name = "address.streetLine1",
-          column = @Column(name = "addr_i18n_street_line1")),
-      @AttributeOverride(
-          name = "address.streetLine2",
-          column = @Column(name = "addr_i18n_street_line2")),
-      @AttributeOverride(
-          name = "address.streetLine3",
-          column = @Column(name = "addr_i18n_street_line3")),
-      @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
-      @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
-      @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
-      @AttributeOverride(
-          name = "address.countryCode",
-          column = @Column(name = "addr_i18n_country_code"))
+    @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_i18n_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_i18n_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_i18n_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_i18n_country_code"))
   })
   PostalInfo internationalizedPostalInfo;
 
@@ -130,8 +131,8 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
   @IgnoreSave(IfNull.class)
   @Embedded
   @AttributeOverrides({
-      @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
-      @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
   })
   ContactPhoneNumber voice;
 
@@ -139,8 +140,8 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
   @IgnoreSave(IfNull.class)
   @Embedded
   @AttributeOverrides({
-      @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
-      @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
   })
   ContactPhoneNumber fax;
 
@@ -151,8 +152,8 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
   /** Authorization info (aka transfer secret) of the contact. */
   @Embedded
   @AttributeOverrides({
-      @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
-      @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
+    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
+    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
   })
   ContactAuthInfo authInfo;
 
@@ -172,13 +173,13 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
   /** Disclosure policy. */
   @Embedded
   @AttributeOverrides({
-      @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
-      @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
-      @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
-      @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
-      @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
-      @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
-      @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
+    @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
+    @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
+    @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
+    @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
+    @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
+    @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
   })
   Disclose disclose;
 
@@ -224,12 +225,12 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
     return disclose;
   }
 
-  public final String getCurrentSponsorClientId() {
+  public String getCurrentSponsorClientId() {
     return getPersistedCurrentSponsorClientId();
   }
 
   @Override
-  public final ContactTransferData getTransferData() {
+  public ContactTransferData getTransferData() {
     return Optional.ofNullable(transferData).orElse(ContactTransferData.EMPTY);
   }
 

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -37,11 +37,11 @@ import org.joda.time.DateTime;
 @javax.persistence.Table(
     name = "Contact",
     indexes = {
-        @javax.persistence.Index(columnList = "creationTime"),
-        @javax.persistence.Index(columnList = "currentSponsorRegistrarId"),
-        @javax.persistence.Index(columnList = "deletionTime"),
-        @javax.persistence.Index(columnList = "contactId", unique = true),
-        @javax.persistence.Index(columnList = "searchName")
+      @javax.persistence.Index(columnList = "creationTime"),
+      @javax.persistence.Index(columnList = "currentSponsorRegistrarId"),
+      @javax.persistence.Index(columnList = "deletionTime"),
+      @javax.persistence.Index(columnList = "contactId", unique = true),
+      @javax.persistence.Index(columnList = "searchName")
     })
 @ExternalMessagingName("contact")
 @WithStringVKey

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -227,7 +227,8 @@ public class DomainBase extends EppResource
    * refer to a {@link PollMessage} timed to when the domain is fully deleted. If the domain is
    * restored, the message should be deleted.
    */
-  @Transient Key<PollMessage.OneTime> deletePollMessage;
+  @Column(name = "deletion_poll_message_id")
+  VKey<PollMessage.OneTime> deletePollMessage;
 
   /**
    * The recurring billing event associated with this domain's autorenewals.
@@ -237,7 +238,8 @@ public class DomainBase extends EppResource
    * {@link #registrationExpirationTime} is changed the recurrence should be closed, a new one
    * should be created, and this field should be updated to point to the new one.
    */
-  @Transient Key<BillingEvent.Recurring> autorenewBillingEvent;
+  @Column(name = "billing_recurrence_id")
+  VKey<BillingEvent.Recurring> autorenewBillingEvent;
 
   /**
    * The recurring poll message associated with this domain's autorenewals.
@@ -247,7 +249,8 @@ public class DomainBase extends EppResource
    * {@link #registrationExpirationTime} is changed the recurrence should be closed, a new one
    * should be created, and this field should be updated to point to the new one.
    */
-  @Transient Key<PollMessage.Autorenew> autorenewPollMessage;
+  @Column(name = "autorenew_poll_message_id")
+  VKey<PollMessage.Autorenew> autorenewPollMessage;
 
   /** The unexpired grace periods for this domain (some of which may not be active yet). */
   @Transient @ElementCollection Set<GracePeriod> gracePeriods;
@@ -316,15 +319,15 @@ public class DomainBase extends EppResource
     return registrationExpirationTime;
   }
 
-  public Key<PollMessage.OneTime> getDeletePollMessage() {
+  public VKey<PollMessage.OneTime> getDeletePollMessage() {
     return deletePollMessage;
   }
 
-  public Key<BillingEvent.Recurring> getAutorenewBillingEvent() {
+  public VKey<BillingEvent.Recurring> getAutorenewBillingEvent() {
     return autorenewBillingEvent;
   }
 
-  public Key<PollMessage.Autorenew> getAutorenewPollMessage() {
+  public VKey<PollMessage.Autorenew> getAutorenewPollMessage() {
     return autorenewPollMessage;
   }
 
@@ -453,14 +456,8 @@ public class DomainBase extends EppResource
               .setRegistrationExpirationTime(expirationDate)
               // Set the speculatively-written new autorenew events as the domain's autorenew
               // events.
-              .setAutorenewBillingEvent(
-                  transferData.getServerApproveAutorenewEvent() == null
-                      ? null
-                      : transferData.getServerApproveAutorenewEvent().getOfyKey())
-              .setAutorenewPollMessage(
-                  transferData.getServerApproveAutorenewPollMessage() == null
-                      ? null
-                      : transferData.getServerApproveAutorenewPollMessage().getOfyKey());
+              .setAutorenewBillingEvent(transferData.getServerApproveAutorenewEvent())
+              .setAutorenewPollMessage(transferData.getServerApproveAutorenewPollMessage());
       if (transferData.getTransferPeriod().getValue() == 1) {
         // Set the grace period using a key to the prescheduled transfer billing event.  Not using
         // GracePeriod.forBillingEvent() here in order to avoid the actual Datastore fetch.
@@ -471,9 +468,7 @@ public class DomainBase extends EppResource
                     transferExpirationTime.plus(
                         Registry.get(getTld()).getTransferGracePeriodLength()),
                     transferData.getGainingClientId(),
-                    transferData.getServerApproveBillingEvent() == null
-                        ? null
-                        : transferData.getServerApproveBillingEvent().getOfyKey())));
+                    transferData.getServerApproveBillingEvent())));
       } else {
         // There won't be a billing event, so we don't need a grace period
         builder.setGracePeriods(ImmutableSet.of());
@@ -801,19 +796,17 @@ public class DomainBase extends EppResource
       return this;
     }
 
-    public Builder setDeletePollMessage(Key<PollMessage.OneTime> deletePollMessage) {
+    public Builder setDeletePollMessage(VKey<PollMessage.OneTime> deletePollMessage) {
       getInstance().deletePollMessage = deletePollMessage;
       return this;
     }
 
-    public Builder setAutorenewBillingEvent(
-        Key<BillingEvent.Recurring> autorenewBillingEvent) {
+    public Builder setAutorenewBillingEvent(VKey<BillingEvent.Recurring> autorenewBillingEvent) {
       getInstance().autorenewBillingEvent = autorenewBillingEvent;
       return this;
     }
 
-    public Builder setAutorenewPollMessage(
-        Key<PollMessage.Autorenew> autorenewPollMessage) {
+    public Builder setAutorenewPollMessage(VKey<PollMessage.Autorenew> autorenewPollMessage) {
       getInstance().autorenewPollMessage = autorenewPollMessage;
       return this;
     }

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -17,12 +17,13 @@ package google.registry.model.domain;
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.persistence.VKey;
 import javax.annotation.Nullable;
 import javax.persistence.Column;
 import javax.persistence.GeneratedValue;
@@ -57,18 +58,18 @@ public class GracePeriod extends ImmutableObject {
 
   /**
    * The one-time billing event corresponding to the action that triggered this grace period, or
-   * null if not applicable.  Not set for autorenew grace periods (which instead use the field
-   * {@code billingEventRecurring}) or for redemption grace periods (since deletes have no cost).
+   * null if not applicable. Not set for autorenew grace periods (which instead use the field {@code
+   * billingEventRecurring}) or for redemption grace periods (since deletes have no cost).
    */
   // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  Key<BillingEvent.OneTime> billingEventOneTime = null;
+  VKey<BillingEvent.OneTime> billingEventOneTime = null;
 
   /**
    * The recurring billing event corresponding to the action that triggered this grace period, if
    * applicable - i.e. if the action was an autorenew - or null in all other cases.
    */
   // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  Key<BillingEvent.Recurring> billingEventRecurring = null;
+  VKey<BillingEvent.Recurring> billingEventRecurring = null;
 
   public GracePeriodStatus getType() {
     return type;
@@ -91,8 +92,7 @@ public class GracePeriod extends ImmutableObject {
    * Returns the one time billing event. The value will only be non-null if the type of this grace
    * period is not AUTO_RENEW.
    */
-
-  public Key<BillingEvent.OneTime> getOneTimeBillingEvent() {
+  public VKey<BillingEvent.OneTime> getOneTimeBillingEvent() {
     return billingEventOneTime;
   }
 
@@ -100,16 +100,16 @@ public class GracePeriod extends ImmutableObject {
    * Returns the recurring billing event. The value will only be non-null if the type of this grace
    * period is AUTO_RENEW.
    */
-  public Key<BillingEvent.Recurring> getRecurringBillingEvent() {
+  public VKey<BillingEvent.Recurring> getRecurringBillingEvent() {
     return billingEventRecurring;
   }
 
   private static GracePeriod createInternal(
-       GracePeriodStatus type,
-       DateTime expirationTime,
-       String clientId,
-       @Nullable Key<BillingEvent.OneTime> billingEventOneTime,
-       @Nullable Key<BillingEvent.Recurring> billingEventRecurring) {
+      GracePeriodStatus type,
+      DateTime expirationTime,
+      String clientId,
+      @Nullable VKey<BillingEvent.OneTime> billingEventOneTime,
+      @Nullable VKey<BillingEvent.Recurring> billingEventRecurring) {
     checkArgument((billingEventOneTime == null) || (billingEventRecurring == null),
         "A grace period can have at most one billing event");
     checkArgument(
@@ -127,15 +127,15 @@ public class GracePeriod extends ImmutableObject {
   /**
    * Creates a GracePeriod for an (optional) OneTime billing event.
    *
-   * <p>Normal callers should always use {@link #forBillingEvent} instead, assuming they do not
-   * need to avoid loading the BillingEvent from Datastore.  This method should typically be
-   * called only from test code to explicitly construct GracePeriods.
+   * <p>Normal callers should always use {@link #forBillingEvent} instead, assuming they do not need
+   * to avoid loading the BillingEvent from Datastore. This method should typically be called only
+   * from test code to explicitly construct GracePeriods.
    */
   public static GracePeriod create(
       GracePeriodStatus type,
       DateTime expirationTime,
       String clientId,
-      @Nullable Key<BillingEvent.OneTime> billingEventOneTime) {
+      @Nullable VKey<BillingEvent.OneTime> billingEventOneTime) {
     return createInternal(type, expirationTime, clientId, billingEventOneTime, null);
   }
 
@@ -144,7 +144,7 @@ public class GracePeriod extends ImmutableObject {
       GracePeriodStatus type,
       DateTime expirationTime,
       String clientId,
-      Key<BillingEvent.Recurring> billingEventRecurring) {
+      VKey<Recurring> billingEventRecurring) {
     checkArgumentNotNull(billingEventRecurring, "billingEventRecurring cannot be null");
     return createInternal(type, expirationTime, clientId, null, billingEventRecurring);
   }
@@ -159,6 +159,6 @@ public class GracePeriod extends ImmutableObject {
   public static GracePeriod forBillingEvent(
       GracePeriodStatus type, BillingEvent.OneTime billingEvent) {
     return create(
-        type, billingEvent.getBillingTime(), billingEvent.getClientId(), Key.create(billingEvent));
+        type, billingEvent.getBillingTime(), billingEvent.getClientId(), billingEvent.createVKey());
   }
 }

core/src/main/java/google/registry/model/ofy/CommitLoggedWork.java

@@ -168,7 +168,7 @@ class CommitLoggedWork<R> implements Runnable {
       DateTime transactionTime, Set<Entry<Key<BackupGroupRoot>, BackupGroupRoot>> bgrEntries) {
     ImmutableMap.Builder<Key<BackupGroupRoot>, DateTime> builder = new ImmutableMap.Builder<>();
     for (Entry<Key<BackupGroupRoot>, BackupGroupRoot> entry : bgrEntries) {
-      DateTime updateTime = entry.getValue().getUpdateAutoTimestamp().getTimestamp();
+      DateTime updateTime = entry.getValue().getUpdateTimestamp().getTimestamp();
       if (!updateTime.isBefore(transactionTime)) {
         builder.put(entry.getKey(), updateTime);
       }

core/src/main/java/google/registry/model/rde/RdeRevision.java

@@ -48,6 +48,10 @@ public final class RdeRevision extends ImmutableObject {
    */
   int revision;
 
+  public int getRevision() {
+    return revision;
+  }
+
   /**
    * Returns next revision ID to use when staging a new deposit file for the given triplet.
    *

core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java

@@ -15,6 +15,7 @@
 package google.registry.model.registry.label;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
@@ -30,6 +31,7 @@ import com.google.common.collect.Multiset;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
@@ -42,6 +44,11 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -49,25 +56,46 @@ import org.joda.time.DateTime;
  *
  * @param <T> The type of the root value being listed, e.g. {@link ReservationType}.
  * @param <R> The type of domain label entry being listed, e.g. {@link ReservedListEntry} (note,
- *            must subclass {@link DomainLabelEntry}.
+ *     must subclass {@link DomainLabelEntry}.
  */
+@MappedSuperclass
 public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends DomainLabelEntry<T, ?>>
     extends ImmutableObject implements Buildable {
 
+  @Ignore
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  Long revisionId;
+
   @Id
+  @Column(nullable = false)
   String name;
 
-  @Parent
-  Key<EntityGroupRoot> parent = getCrossTldKey();
+  @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
-  DateTime creationTime;
+  @Transient DateTime creationTime;
 
+  // The list in Cloud SQL is immutable, we only have a creation_timestamp field and it should be
+  // set to the timestamp when the list is created. In Datastore, we have two fields and the
+  // lastUpdateTime is set to the current timestamp when creating and updating a list. So, we use
+  // lastUpdateTime as the creation_timestamp column during the dual-write phase for compatibility.
+  @Column(name = "creation_timestamp", nullable = false)
   DateTime lastUpdateTime;
 
+  /** Returns the ID of this revision, or throws if null. */
+  public long getRevisionId() {
+    checkState(
+        revisionId != null,
+        "revisionId is null because this object has not been persisted to the database yet");
+    return revisionId;
+  }
+
+  /** Returns the name of the reserved list. */
   public String getName() {
     return name;
   }
 
+  /** Returns the creation time of this revision of the reserved list. */
   public DateTime getCreationTime() {
     return creationTime;
   }
@@ -183,6 +211,9 @@ public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends Dom
     @Override
     public T build() {
       checkArgument(!isNullOrEmpty(getInstance().name), "List must have a name");
+      // The list is immutable in Cloud SQL, so make sure the revision id is not set when the
+      // builder object is created from a list object
+      getInstance().revisionId = null;
       return super.build();
     }
   }

core/src/main/java/google/registry/model/registry/label/DomainLabelEntry.java

@@ -23,16 +23,20 @@ import com.google.common.net.InternetDomainName;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
+import javax.persistence.Column;
+import javax.persistence.MappedSuperclass;
 
 /**
  * Represents a label entry parsed from a line in a reserved/premium list txt file.
  *
  * @param <T> The type of the value stored for the domain label, e.g. {@link ReservationType}.
  */
+@MappedSuperclass
 public abstract class DomainLabelEntry<T extends Comparable<?>, D extends DomainLabelEntry<?, ?>>
     extends ImmutableObject implements Comparable<D> {
 
   @Id
+  @Column(name = "domain_label", insertable = false, updatable = false)
   String label;
 
   String comment;

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -14,7 +14,9 @@
 
 package google.registry.model.registry.label;
 
+import static com.google.common.base.Charsets.US_ASCII;
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.hash.Funnels.stringFunnel;
 import static com.google.common.hash.Funnels.unencodedCharsFunnel;
 import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
 import static google.registry.config.RegistryConfig.getSingletonCachePersistDuration;
@@ -32,43 +34,82 @@ import com.google.common.cache.CacheLoader;
 import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.hash.BloomFilter;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registry.Registry;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.tld.PremiumListDao;
 import google.registry.util.NonFinalForTesting;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import javax.annotation.Nullable;
+import javax.persistence.CollectionTable;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.MapKeyColumn;
+import javax.persistence.PostLoad;
+import javax.persistence.PrePersist;
+import javax.persistence.Table;
+import javax.persistence.Transient;
+import org.hibernate.LazyInitializationException;
+import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.Duration;
 
-/** A premium list entity, persisted to Datastore, that is used to check domain label prices. */
+/**
+ * A premium list entity that is used to check domain label prices.
+ *
+ * <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
+ * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
+ * succeeds, we will end up with having two exact same premium lists that differ only by revisionId.
+ * This is fine though, because we only use the list with the highest revisionId.
+ */
 @ReportedOn
 @Entity
+@javax.persistence.Entity
+@Table(indexes = {@Index(columnList = "name", name = "premiumlist_name_idx")})
 public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.PremiumListEntry>
-    implements DatastoreEntity {
+    implements DatastoreAndSqlEntity {
 
   /** Stores the revision key for the set of currently used premium list entry entities. */
-  Key<PremiumListRevision> revisionKey;
+  @Transient Key<PremiumListRevision> revisionKey;
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // PremiumList is dual-written
-  }
+  @Ignore
+  @Column(nullable = false)
+  CurrencyUnit currency;
+
+  @Ignore
+  @ElementCollection
+  @CollectionTable(
+      name = "PremiumEntry",
+      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
+  @MapKeyColumn(name = "domainLabel")
+  @Column(name = "price", nullable = false)
+  Map<String, BigDecimal> labelsToPrices;
+
+  @Ignore
+  @Column(nullable = false)
+  BloomFilter<String> bloomFilter;
 
   /** Virtual parent entity for premium list entry entities associated with a single revision. */
   @ReportedOn
@@ -247,6 +288,35 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
     return Optional.ofNullable(loadPremiumList(name));
   }
 
+  /** Returns the {@link CurrencyUnit} used for this list. */
+  public CurrencyUnit getCurrency() {
+    return currency;
+  }
+
+  /**
+   * Returns a {@link Map} of domain labels to prices.
+   *
+   * <p>Note that this is lazily loaded and thus will throw a {@link LazyInitializationException} if
+   * used outside the transaction in which the given entity was loaded. You generally should not be
+   * using this anyway as it's inefficient to load all of the PremiumEntry rows if you don't need
+   * them. To check prices, use {@link PremiumListDao#getPremiumPrice} instead.
+   */
+  @Nullable
+  public ImmutableMap<String, BigDecimal> getLabelsToPrices() {
+    return labelsToPrices == null ? null : ImmutableMap.copyOf(labelsToPrices);
+  }
+
+  /**
+   * Returns a Bloom filter to determine whether a label might be premium, or is definitely not.
+   *
+   * <p>If the domain label might be premium, then the next step is to check for the existence of a
+   * corresponding row in the PremiumListEntry table. Otherwise, we know for sure it's not premium,
+   * and no DB load is required.
+   */
+  public BloomFilter<String> getBloomFilter() {
+    return bloomFilter;
+  }
+
   /**
    * A premium list entry entity, persisted to Datastore. Each instance represents the price of a
    * single label on a given TLD.
@@ -339,9 +409,39 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
       return this;
     }
 
+    public Builder setCurrency(CurrencyUnit currency) {
+      getInstance().currency = currency;
+      return this;
+    }
+
+    public Builder setLabelsToPrices(Map<String, BigDecimal> labelsToPrices) {
+      getInstance().labelsToPrices = ImmutableMap.copyOf(labelsToPrices);
+      return this;
+    }
+
     @Override
     public PremiumList build() {
+      if (getInstance().labelsToPrices != null) {
+        // ASCII is used for the charset because all premium list domain labels are stored
+        // punycoded.
+        getInstance().bloomFilter =
+            BloomFilter.create(stringFunnel(US_ASCII), getInstance().labelsToPrices.size());
+        getInstance()
+            .labelsToPrices
+            .keySet()
+            .forEach(label -> getInstance().bloomFilter.put(label));
+      }
       return super.build();
     }
   }
+
+  @PrePersist
+  void prePersist() {
+    lastUpdateTime = creationTime;
+  }
+
+  @PostLoad
+  void postLoad() {
+    creationTime = lastUpdateTime;
+  }
 }

core/src/main/java/google/registry/model/registry/label/ReservationType.java

@@ -38,10 +38,10 @@ public enum ReservationType {
   ALLOWED_IN_SUNRISE("Reserved", 0),
 
   /** The domain can only be registered by providing a specific token. */
-  RESERVED_FOR_SPECIFIC_USE("Allocation token required", 1),
+  RESERVED_FOR_SPECIFIC_USE("Reserved; alloc. token required", 1),
 
   /** The domain is for an anchor tenant and can only be registered using a specific token. */
-  RESERVED_FOR_ANCHOR_TENANT("Allocation token required", 2),
+  RESERVED_FOR_ANCHOR_TENANT("Reserved; alloc. token required", 2),
 
   /**
    * The domain can only be registered during sunrise for defensive purposes, and will never

core/src/main/java/google/registry/model/registry/label/ReservedList.java

@@ -16,11 +16,8 @@ package google.registry.model.registry.label;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
-import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -30,12 +27,8 @@ import com.google.common.base.Splitter;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
@@ -45,45 +38,58 @@ import com.googlecode.objectify.mapper.Mapper;
 import google.registry.model.Buildable;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.label.DomainLabelMetrics.MetricsReservedListMatch;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
-import google.registry.schema.tld.ReservedList.ReservedEntry;
-import google.registry.schema.tld.ReservedListDao;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import javax.annotation.Nullable;
+import javax.persistence.CollectionTable;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.Embeddable;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.MapKeyColumn;
+import javax.persistence.Table;
 import org.joda.time.DateTime;
 
 /**
- * A reserved list entity, persisted to Datastore, that is used to check domain label reservations.
+ * A list of reserved domain labels that are blocked from being registered for various reasons.
+ *
+ * <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
+ * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
+ * succeeds, we will end up with having two exact same reserved lists that differ only by
+ * revisionId. This is fine though, because we only use the list with the highest revisionId.
  */
 @Entity
+@javax.persistence.Entity
+@Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")})
 public final class ReservedList
     extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry>
-    implements DatastoreEntity {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    implements DatastoreAndSqlEntity {
 
   @Mapify(ReservedListEntry.LabelMapper.class)
+  @ElementCollection
+  @CollectionTable(
+      name = "ReservedEntry",
+      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
+  @MapKeyColumn(name = "domain_label")
   Map<String, ReservedListEntry> reservedListMap;
 
+  @Column(nullable = false)
   boolean shouldPublish = true;
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // ReservedList is dual-written
-  }
-
   /**
    * A reserved list entry entity, persisted to Datastore, that represents a single label and its
    * reservation type.
    */
   @Embed
-  public static class ReservedListEntry
-      extends DomainLabelEntry<ReservationType, ReservedListEntry> implements Buildable {
+  @Embeddable
+  public static class ReservedListEntry extends DomainLabelEntry<ReservationType, ReservedListEntry>
+      implements Buildable {
 
+    @Column(nullable = false)
     ReservationType reservationType;
 
     /** Mapper for use with @Mapify */
@@ -150,6 +156,7 @@ public final class ReservedList
     return shouldPublish;
   }
 
+  /** Returns a {@link Map} of domain labels to {@link ReservedListEntry}. */
   public ImmutableMap<String, ReservedListEntry> getReservedListEntries() {
     return ImmutableMap.copyOf(nullToEmpty(reservedListMap));
   }
@@ -239,65 +246,10 @@ public final class ReservedList
               new CacheLoader<String, ReservedList>() {
                 @Override
                 public ReservedList load(String listName) {
-                  ReservedList datastoreList =
-                      ofy()
-                          .load()
-                          .type(ReservedList.class)
-                          .parent(getCrossTldKey())
-                          .id(listName)
-                          .now();
-                  // Also load the list from Cloud SQL, compare the two lists, and log if different.
-                  try {
-                    loadAndCompareCloudSqlList(datastoreList);
-                  } catch (Throwable t) {
-                    logger.atSevere().withCause(t).log("Error comparing reserved lists.");
-                  }
-                  return datastoreList;
+                  return ReservedListDualWriteDao.getLatestRevision(listName).orElse(null);
                 }
               });
 
-  private static final void loadAndCompareCloudSqlList(ReservedList datastoreList) {
-    Optional<google.registry.schema.tld.ReservedList> maybeCloudSqlList =
-        ReservedListDao.getLatestRevision(datastoreList.getName());
-    if (maybeCloudSqlList.isPresent()) {
-      Map<String, ReservedEntry> datastoreLabelsToReservations =
-          datastoreList.reservedListMap.entrySet().parallelStream()
-              .collect(
-                  toImmutableMap(
-                      Map.Entry::getKey,
-                      entry ->
-                          ReservedEntry.create(
-                              entry.getValue().reservationType, entry.getValue().comment)));
-
-      google.registry.schema.tld.ReservedList cloudSqlList = maybeCloudSqlList.get();
-      MapDifference<String, ReservedEntry> diff =
-          Maps.difference(datastoreLabelsToReservations, cloudSqlList.getLabelsToReservations());
-      if (!diff.areEqual()) {
-        if (diff.entriesDiffering().size() > 10) {
-          logger.atWarning().log(
-              String.format(
-                  "Unequal reserved lists detected, Cloud SQL list with revision"
-                      + " id %d has %d different records than the current"
-                      + " Datastore list.",
-                  cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
-        } else {
-          StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
-          diff.entriesDiffering()
-              .forEach(
-                  (label, valueDiff) ->
-                      diffMessage.append(
-                          String.format(
-                              "Domain label %s has entry %s in Datastore and entry"
-                                  + " %s in Cloud SQL.\n",
-                              label, valueDiff.leftValue(), valueDiff.rightValue())));
-          logger.atWarning().log(diffMessage.toString());
-        }
-      }
-    } else {
-      logger.atWarning().log("Reserved list in Cloud SQL is empty.");
-    }
-  }
-
   /**
    * Gets the {@link ReservationType} of a label in a single ReservedList, or returns an absent
    * Optional if none exists in the list.

core/src/main/java/google/registry/model/registry/label/ReservedListDualWriteDao.java

@@ -0,0 +1,124 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.registry.label;
+
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.MapDifference.ValueDifference;
+import com.google.common.collect.Maps;
+import com.google.common.flogger.FluentLogger;
+import com.googlecode.objectify.Key;
+import google.registry.model.registry.label.ReservedList.ReservedListEntry;
+import google.registry.persistence.VKey;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A {@link ReservedList} DAO that does dual-write and dual-read against Datastore and Cloud SQL. It
+ * still uses Datastore as the primary storage and suppresses any exception thrown by Cloud SQL.
+ *
+ * <p>TODO(b/160993806): Delete this DAO and switch to use the SQL only DAO after migrating to Cloud
+ * SQL.
+ */
+public class ReservedListDualWriteDao {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private ReservedListDualWriteDao() {}
+
+  /** Persist a new reserved list to Cloud SQL. */
+  public static void save(ReservedList reservedList) {
+    ofyTm().transact(() -> ofyTm().saveNewOrUpdate(reservedList));
+    try {
+      logger.atInfo().log("Saving reserved list %s to Cloud SQL", reservedList.getName());
+      ReservedListSqlDao.save(reservedList);
+      logger.atInfo().log(
+          "Saved reserved list %s with %d entries to Cloud SQL",
+          reservedList.getName(), reservedList.getReservedListEntries().size());
+    } catch (Throwable t) {
+      logger.atSevere().withCause(t).log("Error saving the reserved list to Cloud SQL.");
+    }
+  }
+
+  /**
+   * Returns the most recent revision of the {@link ReservedList} with the specified name, if it
+   * exists.
+   */
+  public static Optional<ReservedList> getLatestRevision(String reservedListName) {
+    Optional<ReservedList> maybeDatastoreList =
+        ofyTm()
+            .maybeLoad(
+                VKey.createOfy(
+                    ReservedList.class,
+                    Key.create(getCrossTldKey(), ReservedList.class, reservedListName)));
+    try {
+      // Also load the list from Cloud SQL, compare the two lists, and log if different.
+      maybeDatastoreList.ifPresent(ReservedListDualWriteDao::loadAndCompareCloudSqlList);
+    } catch (Throwable t) {
+      logger.atSevere().withCause(t).log("Error comparing reserved lists.");
+    }
+    return maybeDatastoreList;
+  }
+
+  private static void loadAndCompareCloudSqlList(ReservedList datastoreList) {
+    Optional<ReservedList> maybeCloudSqlList =
+        ReservedListSqlDao.getLatestRevision(datastoreList.getName());
+    if (maybeCloudSqlList.isPresent()) {
+      Map<String, ReservedListEntry> datastoreLabelsToReservations =
+          datastoreList.reservedListMap.entrySet().parallelStream()
+              .collect(
+                  toImmutableMap(
+                      Map.Entry::getKey,
+                      entry ->
+                          ReservedListEntry.create(
+                              entry.getKey(),
+                              entry.getValue().reservationType,
+                              entry.getValue().comment)));
+
+      ReservedList cloudSqlList = maybeCloudSqlList.get();
+      MapDifference<String, ReservedListEntry> diff =
+          Maps.difference(datastoreLabelsToReservations, cloudSqlList.reservedListMap);
+      if (!diff.areEqual()) {
+        if (diff.entriesDiffering().size() > 10) {
+          logger.atWarning().log(
+              String.format(
+                  "Unequal reserved lists detected, Cloud SQL list with revision"
+                      + " id %d has %d different records than the current"
+                      + " Datastore list.",
+                  cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
+        } else {
+          StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
+          diff.entriesDiffering().entrySet().stream()
+              .forEach(
+                  entry -> {
+                    String label = entry.getKey();
+                    ValueDifference<ReservedListEntry> valueDiff = entry.getValue();
+                    diffMessage.append(
+                        String.format(
+                            "Domain label %s has entry %s in Datastore and entry"
+                                + " %s in Cloud SQL.\n",
+                            label, valueDiff.leftValue(), valueDiff.rightValue()));
+                  });
+          logger.atWarning().log(diffMessage.toString());
+        }
+      }
+    } else {
+      logger.atWarning().log("Reserved list in Cloud SQL is empty.");
+    }
+  }
+}

core/src/main/java/google/registry/model/registry/label/ReservedListSqlDao.java

@@ -12,20 +12,46 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.schema.tld;
+package google.registry.model.registry.label;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.util.concurrent.UncheckedExecutionException;
 import java.util.Optional;
-import java.util.concurrent.ExecutionException;
 
-/** Data access object class for {@link ReservedList} */
-public class ReservedListDao {
+/**
+ * A {@link ReservedList} DAO for Cloud SQL.
+ *
+ * <p>TODO(b/160993806): Rename this class to ReservedListDao after migrating to Cloud SQL.
+ */
+public class ReservedListSqlDao {
+
+  private ReservedListSqlDao() {}
 
   /** Persist a new reserved list to Cloud SQL. */
   public static void save(ReservedList reservedList) {
-    jpaTm().transact(() -> jpaTm().getEntityManager().persist(reservedList));
+    checkArgumentNotNull(reservedList, "Must specify reservedList");
+    jpaTm().transact(() -> jpaTm().saveNew(reservedList));
+  }
+
+  /**
+   * Returns the most recent revision of the {@link ReservedList} with the specified name, if it
+   * exists.
+   */
+  public static Optional<ReservedList> getLatestRevision(String reservedListName) {
+    return jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .getEntityManager()
+                    .createQuery(
+                        "FROM ReservedList rl LEFT JOIN FETCH rl.reservedListMap WHERE"
+                            + " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl"
+                            + " WHERE subrl.name = :name)",
+                        ReservedList.class)
+                    .setParameter("name", reservedListName)
+                    .getResultStream()
+                    .findFirst());
   }
 
   /**
@@ -46,39 +72,4 @@ public class ReservedListDao {
                         .size()
                     > 0);
   }
-
-  /**
-   * Returns the most recent revision of the {@link ReservedList} with the specified name, if it
-   * exists. TODO(shicong): Change this method to package level access after dual-read phase.
-   */
-  public static Optional<ReservedList> getLatestRevision(String reservedListName) {
-    return jpaTm()
-        .transact(
-            () ->
-                jpaTm()
-                    .getEntityManager()
-                    .createQuery(
-                        "FROM ReservedList rl LEFT JOIN FETCH rl.labelsToReservations WHERE"
-                            + " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl"
-                            + " WHERE subrl.name = :name)",
-                        ReservedList.class)
-                    .setParameter("name", reservedListName)
-                    .getResultStream()
-                    .findFirst());
-  }
-
-  /**
-   * Returns the most recent revision of the {@link ReservedList} with the specified name, from
-   * cache.
-   */
-  public static Optional<ReservedList> getLatestRevisionCached(String reservedListName) {
-    try {
-      return ReservedListCache.cacheReservedLists.get(reservedListName);
-    } catch (ExecutionException e) {
-      throw new UncheckedExecutionException(
-          "Could not retrieve reserved list named " + reservedListName, e);
-    }
-  }
-
-  private ReservedListDao() {}
 }

core/src/main/java/google/registry/persistence/JpaRetries.java

@@ -0,0 +1,58 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
+import java.sql.SQLException;
+import java.util.function.Predicate;
+import javax.persistence.OptimisticLockException;
+
+/** Helpers for identifying retriable database operations. */
+public final class JpaRetries {
+
+  private JpaRetries() {}
+
+  private static final ImmutableSet<String> RETRIABLE_TXN_SQL_STATE =
+      ImmutableSet.of(
+          "40001", // serialization_failure
+          "40P01", // deadlock_detected, PSQL-specific
+          "55006", // object_in_use, PSQL and DB2
+          "55P03" // lock_not_available, PSQL-specific
+          );
+
+  private static final Predicate<Throwable> RETRIABLE_TXN_PREDICATE =
+      Predicates.or(
+          OptimisticLockException.class::isInstance,
+          e ->
+              e instanceof SQLException
+                  && RETRIABLE_TXN_SQL_STATE.contains(((SQLException) e).getSQLState()));
+
+  public static boolean isFailedTxnRetriable(Throwable throwable) {
+    Throwable t = throwable;
+    while (t != null) {
+      if (RETRIABLE_TXN_PREDICATE.test(t)) {
+        return true;
+      }
+      t = t.getCause();
+    }
+    return false;
+  }
+
+  public static boolean isFailedQueryRetriable(Throwable throwable) {
+    // TODO(weiminyu): check for more error codes.
+    return isFailedTxnRetriable(throwable);
+  }
+}

core/src/main/java/google/registry/persistence/NomulusPostgreSQLDialect.java

@@ -13,6 +13,7 @@
 // limitations under the License.
 package google.registry.persistence;
 
+import google.registry.persistence.converter.IntervalDescriptor;
 import google.registry.persistence.converter.StringCollectionDescriptor;
 import google.registry.persistence.converter.StringMapDescriptor;
 import java.sql.Types;
@@ -30,6 +31,7 @@ public class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
     registerColumnType(StringMapDescriptor.COLUMN_TYPE, StringMapDescriptor.COLUMN_NAME);
     registerColumnType(
         StringCollectionDescriptor.COLUMN_TYPE, StringCollectionDescriptor.COLUMN_DDL_NAME);
+    registerColumnType(IntervalDescriptor.COLUMN_TYPE, IntervalDescriptor.COLUMN_NAME);
   }
 
   @Override
@@ -40,5 +42,7 @@ public class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
     typeContributions.contributeSqlTypeDescriptor(StringCollectionDescriptor.getInstance());
     typeContributions.contributeJavaTypeDescriptor(StringMapDescriptor.getInstance());
     typeContributions.contributeSqlTypeDescriptor(StringMapDescriptor.getInstance());
+    typeContributions.contributeJavaTypeDescriptor(IntervalDescriptor.getInstance());
+    typeContributions.contributeSqlTypeDescriptor(IntervalDescriptor.getInstance());
   }
 }

core/src/main/java/google/registry/persistence/VKey.java

@@ -58,6 +58,31 @@ public class VKey<T> extends ImmutableObject implements Serializable {
     return new VKey(kind, null, sqlKey);
   }
 
+  /** Creates a {@link VKey} which only contains the ofy primary key. */
+  public static <T> VKey<T> createOfy(
+      Class<? extends T> kind, com.googlecode.objectify.Key<? extends T> ofyKey) {
+    checkArgumentNotNull(kind, "kind must not be null");
+    checkArgumentNotNull(ofyKey, "ofyKey must not be null");
+    return new VKey(kind, ofyKey, null);
+  }
+
+  /**
+   * Creates a {@link VKey} which only contains the ofy primary key by specifying the id of the
+   * {@link Key}.
+   */
+  public static <T> VKey<T> createOfy(Class<? extends T> kind, long id) {
+    return createOfy(kind, Key.create(kind, id));
+  }
+
+  /**
+   * Creates a {@link VKey} which only contains the ofy primary key by specifying the name of the
+   * {@link Key}.
+   */
+  public static <T> VKey<T> createOfy(Class<? extends T> kind, String name) {
+    checkArgumentNotNull(kind, "name must not be null");
+    return createOfy(kind, Key.create(kind, name));
+  }
+
   /** Creates a {@link VKey} which only contains both sql and ofy primary key. */
   public static <T> VKey<T> create(
       Class<? extends T> kind, Object sqlKey, com.googlecode.objectify.Key ofyKey) {

core/src/main/java/google/registry/persistence/converter/DurationConverter.java

@@ -14,24 +14,67 @@
 
 package google.registry.persistence.converter;
 
+import java.sql.SQLException;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeConverter;
 import javax.persistence.Converter;
 import org.joda.time.Duration;
+import org.joda.time.Period;
+import org.postgresql.util.PGInterval;
 
-/** JPA converter to for storing/retrieving {@link org.joda.time.DateTime} objects. */
+/**
+ * JPA converter to for storing/retrieving {@link org.joda.time.Duration} objects.
+ *
+ * <p>The Joda Time Duration is simply a number of milliseconds representing a length of time. This
+ * can be converted into a PGInterval, but only for the fields that have a standard number of
+ * milliseconds. Therefore, there is no way to populate the months or years field of a PGInterval
+ * and be confident that it is representing the exact number of milliseconds it was intended to
+ * represent.
+ */
 @Converter(autoApply = true)
-public class DurationConverter implements AttributeConverter<Duration, Long> {
+public class DurationConverter implements AttributeConverter<Duration, PGInterval> {
 
   @Override
   @Nullable
-  public Long convertToDatabaseColumn(@Nullable Duration duration) {
-    return duration == null ? null : duration.getMillis();
+  public PGInterval convertToDatabaseColumn(@Nullable Duration duration) {
+    if (duration == null) {
+      return new PGInterval();
+    }
+    PGInterval interval = new PGInterval();
+    Period period = new Period(duration);
+    // For some reason when the period is created from the duration, it does not set days, but
+    // instead just a total number of hours. Years and months are not created because those can
+    // differ in length of milliseconds.
+    interval.setDays(period.getHours() / 24);
+    interval.setHours(period.getHours() % 24);
+    interval.setMinutes(period.getMinutes());
+    double millis = (double) period.getMillis() / 1000;
+    interval.setSeconds(period.getSeconds() + millis);
+    return interval;
   }
 
   @Override
   @Nullable
-  public Duration convertToEntityAttribute(@Nullable Long dbData) {
-    return dbData == null ? null : new Duration(dbData);
+  public Duration convertToEntityAttribute(@Nullable PGInterval dbData) {
+    if (dbData == null) {
+      return null;
+    }
+    PGInterval interval = null;
+    try {
+      interval = new PGInterval(dbData.toString());
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+
+    if (interval.equals(new PGInterval())) {
+      return null;
+    }
+
+    final int days = interval.getDays();
+    final int hours = interval.getHours();
+    final int mins = interval.getMinutes();
+    final int secs = (int) interval.getSeconds();
+    final int millis = interval.getMicroSeconds() / 1000;
+    return new Period(0, 0, 0, days, hours, mins, secs, millis).toStandardDuration();
   }
 }

core/src/main/java/google/registry/persistence/converter/IntervalDescriptor.java

@@ -0,0 +1,139 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import java.sql.CallableStatement;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import org.hibernate.type.descriptor.ValueBinder;
+import org.hibernate.type.descriptor.ValueExtractor;
+import org.hibernate.type.descriptor.WrapperOptions;
+import org.hibernate.type.descriptor.java.AbstractTypeDescriptor;
+import org.hibernate.type.descriptor.java.JavaTypeDescriptor;
+import org.hibernate.type.descriptor.spi.JdbcRecommendedSqlTypeMappingContext;
+import org.hibernate.type.descriptor.sql.BasicBinder;
+import org.hibernate.type.descriptor.sql.BasicExtractor;
+import org.hibernate.type.descriptor.sql.SqlTypeDescriptor;
+import org.postgresql.util.PGInterval;
+
+/**
+ * The {@link JavaTypeDescriptor} and {@link SqlTypeDescriptor} for {@link PGInterval}.
+ *
+ * @see <a
+ *     href="https://docs.jboss.org/hibernate/orm/current/userguide/html_single/Hibernate_User_Guide.html#basic-jpa-convert">JPA
+ *     2.1 AttributeConverters</a>
+ */
+public class IntervalDescriptor extends AbstractTypeDescriptor<PGInterval>
+    implements SqlTypeDescriptor {
+  public static final int COLUMN_TYPE = Types.JAVA_OBJECT;
+  public static final String COLUMN_NAME = "interval";
+  private static final IntervalDescriptor INSTANCE = new IntervalDescriptor();
+
+  private IntervalDescriptor() {
+    super(PGInterval.class);
+  }
+
+  public static IntervalDescriptor getInstance() {
+    return INSTANCE;
+  }
+
+  @Override
+  public PGInterval fromString(String string) {
+    throw new UnsupportedOperationException(
+        "Constructing IntervalDescriptor from string is not allowed");
+  }
+
+  @Override
+  public <X> X unwrap(PGInterval value, Class<X> type, WrapperOptions options) {
+    if (value == null) {
+      return null;
+    }
+    if (PGInterval.class.isAssignableFrom(type)) {
+      return (X) value;
+    }
+    throw unknownUnwrap(type);
+  }
+
+  @Override
+  public <X> PGInterval wrap(X value, WrapperOptions options) {
+    if (value == null) {
+      return null;
+    }
+    if (value instanceof PGInterval) {
+      try {
+        return new PGInterval(value.toString());
+      } catch (SQLException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    throw unknownWrap(value.getClass());
+  }
+
+  @Override
+  public int getSqlType() {
+    return COLUMN_TYPE;
+  }
+
+  @Override
+  public SqlTypeDescriptor getJdbcRecommendedSqlType(JdbcRecommendedSqlTypeMappingContext context) {
+    return this;
+  }
+
+  @Override
+  public boolean canBeRemapped() {
+    return false;
+  }
+
+  @Override
+  public <X> ValueBinder<X> getBinder(JavaTypeDescriptor<X> javaTypeDescriptor) {
+    return new BasicBinder<X>(javaTypeDescriptor, this) {
+      @Override
+      protected void doBind(PreparedStatement st, X value, int index, WrapperOptions options)
+          throws SQLException {
+        st.setObject(index, new PGInterval(value.toString()));
+      }
+
+      @Override
+      protected void doBind(CallableStatement st, X value, String name, WrapperOptions options)
+          throws SQLException {
+        st.setObject(name, new PGInterval(value.toString()));
+      }
+    };
+  }
+
+  @Override
+  public <X> ValueExtractor<X> getExtractor(JavaTypeDescriptor<X> javaTypeDescriptor) {
+    return new BasicExtractor<X>(javaTypeDescriptor, this) {
+      @Override
+      protected X doExtract(ResultSet rs, String name, WrapperOptions options) throws SQLException {
+        return javaTypeDescriptor.wrap(rs.getObject(name), options);
+      }
+
+      @Override
+      protected X doExtract(CallableStatement statement, int index, WrapperOptions options)
+          throws SQLException {
+        return javaTypeDescriptor.wrap(statement.getObject(index), options);
+      }
+
+      @Override
+      protected X doExtract(CallableStatement statement, String name, WrapperOptions options)
+          throws SQLException {
+        return javaTypeDescriptor.wrap(statement.getObject(name), options);
+      }
+    };
+  }
+}

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManager.java

@@ -25,4 +25,11 @@ public interface JpaTransactionManager extends TransactionManager {
 
   /** Deletes the entity by its id, throws exception if the entity is not deleted. */
   public abstract <T> void assertDelete(VKey<T> key);
+
+  /**
+   * Releases all resources and shuts down.
+   *
+   * <p>The errorprone check forbids injection of {@link java.io.Closeable} resources.
+   */
+  void teardown();
 }

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig;
 import google.registry.persistence.VKey;
 import google.registry.util.Clock;
 import java.lang.reflect.Field;
@@ -62,6 +63,11 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     this.clock = clock;
   }
 
+  @Override
+  public void teardown() {
+    emf.close();
+  }
+
   @Override
   public EntityManager getEntityManager() {
     if (transactionInfo.get().entityManager == null) {
@@ -96,9 +102,9 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     EntityTransaction txn = txnInfo.entityManager.getTransaction();
     try {
       txn.begin();
-      txnInfo.inTransaction = true;
-      txnInfo.transactionTime = clock.nowUtc();
+      txnInfo.start(clock);
       T result = work.get();
+      txnInfo.recordTransaction();
       txn.commit();
       return result;
     } catch (RuntimeException | Error e) {
@@ -172,6 +178,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     checkArgumentNotNull(entity, "entity must be specified");
     assertInTransaction();
     getEntityManager().persist(entity);
+    transactionInfo.get().addUpdate(entity);
   }
 
   @Override
@@ -186,6 +193,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     checkArgumentNotNull(entity, "entity must be specified");
     assertInTransaction();
     getEntityManager().merge(entity);
+    transactionInfo.get().addUpdate(entity);
   }
 
   @Override
@@ -201,6 +209,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     assertInTransaction();
     checkArgument(checkExists(entity), "Given entity does not exist");
     getEntityManager().merge(entity);
+    transactionInfo.get().addUpdate(entity);
   }
 
   @Override
@@ -292,6 +301,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
         String.format("DELETE FROM %s WHERE %s", entityType.getName(), getAndClause(entityIds));
     Query query = getEntityManager().createQuery(sql);
     entityIds.forEach(entityId -> query.setParameter(entityId.name, entityId.value));
+    transactionInfo.get().addDelete(key);
     return query.executeUpdate();
   }
 
@@ -382,9 +392,23 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     boolean inTransaction = false;
     DateTime transactionTime;
 
+    // Serializable representation of the transaction to be persisted in the Transaction table.
+    Transaction.Builder contentsBuilder;
+
+    /** Start a new transaction. */
+    private void start(Clock clock) {
+      checkArgumentNotNull(clock);
+      inTransaction = true;
+      transactionTime = clock.nowUtc();
+      if (RegistryConfig.getCloudSqlReplicateTransactions()) {
+        contentsBuilder = new Transaction.Builder();
+      }
+    }
+
     private void clear() {
       inTransaction = false;
       transactionTime = null;
+      contentsBuilder = null;
       if (entityManager != null) {
         // Close this EntityManager just let the connection pool be able to reuse it, it doesn't
         // close the underlying database connection.
@@ -392,5 +416,26 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
         entityManager = null;
       }
     }
+
+    private void addUpdate(Object entity) {
+      if (contentsBuilder != null) {
+        contentsBuilder.addUpdate(entity);
+      }
+    }
+
+    private void addDelete(VKey<?> key) {
+      if (contentsBuilder != null) {
+        contentsBuilder.addDelete(key);
+      }
+    }
+
+    private void recordTransaction() {
+      if (contentsBuilder != null) {
+        Transaction persistedTxn = contentsBuilder.build();
+        if (!persistedTxn.isEmpty()) {
+          entityManager.persist(persistedTxn.toEntity());
+        }
+      }
+    }
   }
 }

core/src/main/java/google/registry/persistence/transaction/Transaction.java

@@ -109,11 +109,20 @@ public class Transaction extends ImmutableObject implements Buildable {
     return builder.build();
   }
 
+  /** Returns true if the transaction contains no mutations. */
+  public boolean isEmpty() {
+    return mutations.isEmpty();
+  }
+
   @Override
   public Builder asBuilder() {
     return new Builder(clone(this));
   }
 
+  public final TransactionEntity toEntity() {
+    return new TransactionEntity(serialize());
+  }
+
   public static class Builder extends GenericBuilder<Transaction, Builder> {
 
     ImmutableList.Builder listBuilder = new ImmutableList.Builder();

core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java

@@ -0,0 +1,43 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.transaction;
+
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+/**
+ * Object to be stored in the transaction table.
+ *
+ * <p>This consists of a sequential identifier and a serialized {@code Tranaction} object.
+ */
+@Entity
+@Table(name = "Transaction")
+public class TransactionEntity {
+
+  @Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  long id;
+
+  byte[] contents;
+
+  TransactionEntity() {}
+
+  TransactionEntity(byte[] contents) {
+    this.contents = contents;
+  }
+}

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -75,7 +75,12 @@ public class TransactionManagerFactory {
     return tm;
   }
 
-  /** Returns {@link JpaTransactionManager} instance. */
+  /**
+   * Returns {@link JpaTransactionManager} instance.
+   *
+   * <p>Between invocations of {@link TransactionManagerFactory#setJpaTm} every call to this method
+   * returns the same instance.
+   */
   public static JpaTransactionManager jpaTm() {
     return jpaTm.get();
   }
@@ -93,7 +98,7 @@ public class TransactionManagerFactory {
         RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
             || RegistryToolEnvironment.get() != null,
         "setJpamTm() should only be called by tools and tests.");
-    jpaTm = jpaTmSupplier;
+    jpaTm = Suppliers.memoize(jpaTmSupplier::get);
   }
 
   /** Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}. */

core/src/main/java/google/registry/rde/RdeModule.java

@@ -89,7 +89,7 @@ public abstract class RdeModule {
   @Provides
   @Parameter(PARAM_LENIENT)
   static boolean provideLenient(HttpServletRequest req) {
-    return extractBooleanParameter(req, PARAM_REVISION);
+    return extractBooleanParameter(req, PARAM_LENIENT);
   }
 
   @Provides

core/src/main/java/google/registry/rde/RdeResourceType.java

@@ -29,7 +29,7 @@ public enum RdeResourceType {
   DOMAIN("urn:ietf:params:xml:ns:rdeDomain-1.0", EnumSet.of(FULL, THIN)),
   HOST("urn:ietf:params:xml:ns:rdeHost-1.0", EnumSet.of(FULL)),
   REGISTRAR("urn:ietf:params:xml:ns:rdeRegistrar-1.0", EnumSet.of(FULL, THIN)),
-  IDN("urn:ietf:params:xml:ns:rdeIDN-1.0", EnumSet.of(FULL, THIN)),
+  IDN("urn:ietf:params:xml:ns:rdeIDN-1.0", EnumSet.of(FULL)),
   HEADER("urn:ietf:params:xml:ns:rdeHeader-1.0", EnumSet.of(FULL, THIN));
 
   private final String uri;

core/src/main/java/google/registry/rde/RdeStagingReducer.java

@@ -77,7 +77,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
   private final byte[] stagingKeyBytes;
   private final RdeMarshaller marshaller;
 
-  private RdeStagingReducer(
+  RdeStagingReducer(
       TaskQueueUtils taskQueueUtils,
       LockHandler lockHandler,
       int gcsBufferSize,
@@ -125,7 +125,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
     final DateTime watermark = key.watermark();
     final int revision =
         Optional.ofNullable(key.revision())
-            .orElse(RdeRevision.getNextRevision(tld, watermark, mode));
+            .orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
     String id = RdeUtil.timestampToId(watermark);
     String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
     if (key.manual()) {
@@ -168,9 +168,13 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
           logger.atSevere().log("Fragment error: %s", fragment.error());
         }
       }
-      for (IdnTableEnum idn : IdnTableEnum.values()) {
-        output.write(marshaller.marshalIdn(idn.getTable()));
-        counter.increment(RdeResourceType.IDN);
+
+      // Don't write the IDN elements for BRDA.
+      if (mode == RdeMode.FULL) {
+        for (IdnTableEnum idn : IdnTableEnum.values()) {
+          output.write(marshaller.marshalIdn(idn.getTable()));
+          counter.increment(RdeResourceType.IDN);
+        }
       }
 
       // Output XML that says how many resources were emitted.

core/src/main/java/google/registry/schema/tld/PremiumEntry.java

@@ -16,6 +16,7 @@ package google.registry.schema.tld;
 
 import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
+import google.registry.model.registry.label.PremiumList;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import java.io.Serializable;

core/src/main/java/google/registry/schema/tld/PremiumList.java

@@ -1,151 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.schema.tld;
-
-import static com.google.common.base.Charsets.US_ASCII;
-import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.hash.Funnels.stringFunnel;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.hash.BloomFilter;
-import google.registry.model.CreateAutoTimestamp;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
-import java.math.BigDecimal;
-import java.util.Map;
-import javax.annotation.Nullable;
-import javax.persistence.CollectionTable;
-import javax.persistence.Column;
-import javax.persistence.ElementCollection;
-import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
-import javax.persistence.Id;
-import javax.persistence.Index;
-import javax.persistence.JoinColumn;
-import javax.persistence.MapKeyColumn;
-import javax.persistence.Table;
-import org.hibernate.LazyInitializationException;
-import org.joda.money.CurrencyUnit;
-import org.joda.time.DateTime;
-
-/**
- * A list of premium prices for domain names.
- *
- * <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
- * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
- * succeeds, we will end up with having two exact same premium lists that differ only by revisionId.
- * This is fine though, because we only use the list with the highest revisionId.
- */
-@Entity
-@Table(indexes = {@Index(columnList = "name", name = "premiumlist_name_idx")})
-public class PremiumList implements SqlEntity {
-
-  @Column(nullable = false)
-  private String name;
-
-  @Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
-  @Column(nullable = false)
-  private Long revisionId;
-
-  @Column(nullable = false)
-  private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
-
-  @Column(nullable = false)
-  private CurrencyUnit currency;
-
-  @ElementCollection
-  @CollectionTable(
-      name = "PremiumEntry",
-      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
-  @MapKeyColumn(name = "domainLabel")
-  @Column(name = "price", nullable = false)
-  private Map<String, BigDecimal> labelsToPrices;
-
-  @Column(nullable = false)
-  private BloomFilter<String> bloomFilter;
-
-  private PremiumList(String name, CurrencyUnit currency, Map<String, BigDecimal> labelsToPrices) {
-    this.name = name;
-    this.currency = currency;
-    this.labelsToPrices = labelsToPrices;
-    // ASCII is used for the charset because all premium list domain labels are stored punycoded.
-    this.bloomFilter = BloomFilter.create(stringFunnel(US_ASCII), labelsToPrices.size());
-    labelsToPrices.keySet().forEach(this.bloomFilter::put);
-  }
-
-  // Hibernate requires this default constructor.
-  private PremiumList() {}
-
-  /** Constructs a {@link PremiumList} object. */
-  public static PremiumList create(
-      String name, CurrencyUnit currency, Map<String, BigDecimal> labelsToPrices) {
-    return new PremiumList(name, currency, labelsToPrices);
-  }
-
-  /** Returns the name of the premium list, which is usually also a TLD string. */
-  public String getName() {
-    return name;
-  }
-
-  /** Returns the {@link CurrencyUnit} used for this list. */
-  public CurrencyUnit getCurrency() {
-    return currency;
-  }
-
-  /** Returns the ID of this revision, or throws if null. */
-  public Long getRevisionId() {
-    checkState(
-        revisionId != null,
-        "revisionId is null because this object has not yet been persisted to the DB");
-    return revisionId;
-  }
-
-  /** Returns the creation time of this revision of the premium list. */
-  public DateTime getCreationTimestamp() {
-    return creationTimestamp.getTimestamp();
-  }
-
-  /**
-   * Returns a {@link Map} of domain labels to prices.
-   *
-   * <p>Note that this is lazily loaded and thus will throw a {@link LazyInitializationException} if
-   * used outside the transaction in which the given entity was loaded. You generally should not be
-   * using this anyway as it's inefficient to load all of the PremiumEntry rows if you don't need
-   * them. To check prices, use {@link PremiumListDao#getPremiumPrice} instead.
-   */
-  @Nullable
-  public ImmutableMap<String, BigDecimal> getLabelsToPrices() {
-    return labelsToPrices == null ? null : ImmutableMap.copyOf(labelsToPrices);
-  }
-
-  /**
-   * Returns a Bloom filter to determine whether a label might be premium, or is definitely not.
-   *
-   * <p>If the domain label might be premium, then the next step is to check for the existence of a
-   * corresponding row in the PremiumListEntry table. Otherwise, we know for sure it's not premium,
-   * and no DB load is required.
-   */
-  public BloomFilter<String> getBloomFilter() {
-    return bloomFilter;
-  }
-
-  @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // PremiumList is dual-written
-  }
-}

core/src/main/java/google/registry/schema/tld/PremiumListCache.java

@@ -25,6 +25,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import google.registry.model.registry.label.PremiumList;
 import google.registry.util.NonFinalForTesting;
 import java.math.BigDecimal;
 import java.util.Optional;