Only use asymmetric VKeys for EPP resources (#611)

Given that we currently have no way of reconstituting a symmetric key
from the asymmetric key (or at least, we don't have a 100% reliable way
of doing so) it's best to keep keys as asymmetrical, referring to the
correct database. That way, we don't get situations where we cannot
compare equality of two keys due to one being asymmetrical and one being
symmetrical.

appengine_war.gradle

@@ -31,10 +31,6 @@ def coreResourcesDir = "${rootDir}/core/build/resources/main"
 war {
     webInf {
         from "../../core/src/main/java/google/registry/env/common/${project.name}/WEB-INF"
-
-        from("${coreResourcesDir}/META-INF/persistence.xml") {
-            into "classes/META-INF"
-        }
     }
 }
 

build.gradle

@@ -197,6 +197,10 @@ task runPresubmits(type: Exec) {
   args('config/presubmits.py')
 }
 
+def javadocSource = []
+def javadocClasspath = []
+def javadocDependentTasks = []
+
 subprojects {
   // Skip no-op project
   if (project.name == 'services') return
@@ -317,6 +321,10 @@ subprojects {
       }
     }
   }
+
+  javadocSource << project.sourceSets.main.allJava
+  javadocClasspath << project.sourceSets.main.compileClasspath
+  javadocDependentTasks << project.tasks.compileJava
 }
 
 // If "-P verboseTestOutput=true" is passed in, configure all subprojects to dump all of their
@@ -445,3 +453,19 @@ task javaIncrementalFormatApply {
 }
 
 tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)
+
+task javadoc(type: Javadoc) {
+  source javadocSource
+  classpath = files(javadocClasspath)
+  destinationDir = file("${buildDir}/docs/javadoc")
+  options.encoding = "UTF-8"
+  // In a lot of places we don't write @return so suppress warnings about that.
+  options.addBooleanOption('Xdoclint:all,-missing', true)
+  options.addBooleanOption("-allow-script-in-comments",true)
+  options.tags = ["type:a:Generic Type",
+                  "error:a:Expected Error"]
+}
+
+tasks.build.dependsOn(tasks.javadoc)
+
+javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }

buildSrc/gradle/dependency-locks/testCompile.lockfile

@@ -61,12 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.10.0
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.mockito:mockito-core:2.25.0
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -61,12 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.10.0
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.mockito:mockito-core:2.25.0
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testRuntime.lockfile

@@ -61,12 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.10.0
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.mockito:mockito-core:2.25.0
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -61,12 +61,12 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.10.0
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.mockito:mockito-core:2.25.0
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testCompile.lockfile

@@ -20,10 +20,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.10.0
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -20,10 +20,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.10.0
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testRuntime.lockfile

@@ -21,10 +21,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.10.0
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.opentest4j:opentest4j:1.2.0

common/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -21,10 +21,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.10.0
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.opentest4j:opentest4j:1.2.0

common/src/main/java/google/registry/util/Sleeper.java

@@ -21,7 +21,6 @@ import org.joda.time.ReadableDuration;
  * An object which accepts requests to put the current thread to sleep.
  *
  * @see SystemSleeper
- * @see google.registry.testing.FakeSleeper
  */
 @ThreadSafe
 public interface Sleeper {

config/presubmits.py

@@ -99,6 +99,15 @@ PRESUBMITS = {
         "System.(out|err).println is only allowed in tools/ packages. Please "
         "use a logger instead.",
 
+    # ObjectifyService.register is restricted to main/ or AppEngineRule.
+    PresubmitCheck(
+        r".*\bObjectifyService\.register", "java", {
+            "/build/", "/generated/", "node_modules/", "src/main/",
+            "AppEngineRule.java"
+        }):
+      "ObjectifyService.register is not allowed in tests. Please use "
+      "AppengineRule.register instead.",
+
     # PostgreSQLContainer instantiation must specify docker tag
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",

core/build.gradle

@@ -177,6 +177,8 @@ dependencies {
 
   compile deps['com.beust:jcommander']
   compile deps['com.google.api-client:google-api-client']
+  compile deps['com.google.api-client:google-api-client-appengine']
+  compile deps['com.google.api-client:google-api-client-servlet']
   compile deps['com.google.monitoring-client:metrics']
   compile deps['com.google.monitoring-client:stackdriver']
   compile deps['com.google.api-client:google-api-client-java6']
@@ -216,6 +218,8 @@ dependencies {
   compile deps['com.google.oauth-client:google-oauth-client']
   compile deps['com.google.oauth-client:google-oauth-client-java6']
   compile deps['com.google.oauth-client:google-oauth-client-jetty']
+  compile deps['com.google.oauth-client:google-oauth-client-appengine']
+  compile deps['com.google.oauth-client:google-oauth-client-servlet']
   compile deps['com.google.re2j:re2j']
   compile deps['com.google.template:soy']
   compile deps['com.googlecode.json-simple:json-simple']
@@ -262,6 +266,7 @@ dependencies {
   testCompile deps['org.seleniumhq.selenium:selenium-chrome-driver']
   testCompile deps['org.seleniumhq.selenium:selenium-java']
   testCompile deps['org.seleniumhq.selenium:selenium-remote-driver']
+  runtimeOnly deps['org.slf4j:slf4j-jdk14']
   testCompile deps['org.testcontainers:jdbc']
   compile deps['org.testcontainers:postgresql']
   testCompile deps['org.testcontainers:selenium']
@@ -290,6 +295,8 @@ dependencies {
   testAnnotationProcessor deps['com.google.auto.value:auto-value']
   annotationProcessor deps['com.google.dagger:dagger-compiler']
   testAnnotationProcessor deps['com.google.dagger:dagger-compiler']
+  annotationProcessor project(':processor')
+  testAnnotationProcessor project(':processor')
 
   testCompile deps['com.google.appengine:appengine-testing']
   testCompile deps['com.google.guava:guava-testlib']
@@ -304,6 +311,8 @@ dependencies {
   testCompile deps['org.junit.jupiter:junit-jupiter-api']
   testCompile deps['org.junit.jupiter:junit-jupiter-engine']
   testCompile deps['org.junit.jupiter:junit-jupiter-migrationsupport']
+  testCompile deps['org.junit.platform:junit-platform-runner']
+  testCompile deps['org.junit.platform:junit-platform-suite-api']
   testCompile deps['org.junit.vintage:junit-vintage-engine']
   testCompile deps['org.mockito:mockito-core']
   runtime deps['org.postgresql:postgresql']
@@ -641,6 +650,10 @@ artifacts {
  */
 class FilteringTest extends Test {
 
+  FilteringTest() {
+    useJUnitPlatform();
+  }
+
   private void applyTestFilter() {
     if (project.testFilter) {
       testNameIncludePatterns = project.testFilter.split(',')
@@ -696,7 +709,10 @@ task outcastTest(type: FilteringTest) {
   tests = outcastTestPatterns
 
   // Sets the maximum number of test executors that may exist at the same time.
-  maxParallelForks 5
+  // Note that this number appears to contribute to NoClassDefFoundError
+  // exceptions on certain machines and distros. The root cause is unclear.
+  // Try reducing this number if you experience similar problems.
+  maxParallelForks 3
 }
 
 // Whitebox test verifying that RegistryTool can be instantiated. Note the
@@ -728,6 +744,10 @@ task registryToolIntegrationTest {
 
 // Dedicated test suite for schema-dependent tests.
 task sqlIntegrationTest(type: FilteringTest) {
+  // TestSuite still requires a JUnit 4 runner, which knows how to handle JUnit 5 tests.
+  // Here we need to override parent's choice of JUnit 5. If changing this, remember to
+  // change :integration:sqlIntegrationTest too.
+  useJUnit()
   excludeTestCases = false
   tests = ['google/registry/schema/integration/SqlIntegrationTestSuite.*']
 }
@@ -847,6 +867,8 @@ task standardTest(type: FilteringTest) {
   includeAllTests()
   exclude fragileTestPatterns
   exclude outcastTestPatterns
+  // See SqlIntegrationTestSuite.java
+  exclude '**/*BeforeSuiteTest.*', '**/*AfterSuiteTest.*'
 
   if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") {
     exclude dockerIncompatibleTestPatterns
@@ -858,6 +880,9 @@ task standardTest(type: FilteringTest) {
   // forkEvery 1
 
   // Sets the maximum number of test executors that may exist at the same time.
+  // Also, Gradle executes tests in 1 thread and some of our test infrastructures
+  // depend on that, e.g. DualDatabaseTestInvocationContextProvider injects
+  // different implementation of TransactionManager into TransactionManagerFactory.
   maxParallelForks 5
 
   systemProperty 'test.projectRoot', rootProject.projectRootDir
@@ -879,11 +904,14 @@ createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
 
 // A jar with classes and resources from main sourceSet, excluding internal
 // data. See comments on configurations.nomulus_test above for details.
+// TODO(weiminyu): release process should build this using the public repo to eliminate the need
+// for excludes.
 task nomulusFossJar (type: Jar) {
   archiveBaseName = 'nomulus'
   archiveClassifier = 'public'
   from (project.sourceSets.main.output) {
     exclude 'google/registry/config/files/**'
+    exclude 'google/registry/proxy/config/**'
   }
   from (project.sourceSets.main.output) {
     include 'google/registry/config/files/default-config.yaml'

core/gradle/dependency-locks/annotationProcessor.lockfile

@@ -12,24 +12,24 @@ com.google.dagger:dagger-producers:2.21
 com.google.dagger:dagger-spi:2.21
 com.google.dagger:dagger:2.21
 com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
+com.google.errorprone:error_prone_annotations:2.3.4
 com.google.errorprone:error_prone_check_api:2.3.3
 com.google.errorprone:error_prone_core:2.3.3
 com.google.errorprone:error_prone_type_annotations:2.3.3
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:27.0.1-jre
+com.google.guava:guava:28.2-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.j2objc:j2objc-annotations:1.1
+com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-com.squareup:javapoet:1.11.1
+com.squareup:javapoet:1.12.1
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
+javax.persistence:javax.persistence-api:2.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:2.5.3
+org.checkerframework:checker-qual:2.10.0
 org.checkerframework:dataflow:2.5.3
 org.checkerframework:javacutil:2.5.3
-org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2

core/gradle/dependency-locks/default.lockfile

@@ -235,6 +235,7 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
 org.slf4j:slf4j-api:1.7.28
+org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.12.1
 org.testcontainers:jdbc:1.12.1
 org.testcontainers:postgresql:1.12.1

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -232,6 +232,7 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
 org.slf4j:slf4j-api:1.7.28
+org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.12.1
 org.testcontainers:jdbc:1.12.1
 org.testcontainers:postgresql:1.12.1

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -232,6 +232,7 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
 org.slf4j:slf4j-api:1.7.28
+org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.12.1
 org.testcontainers:jdbc:1.12.1
 org.testcontainers:postgresql:1.12.1

core/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -12,24 +12,24 @@ com.google.dagger:dagger-producers:2.21
 com.google.dagger:dagger-spi:2.21
 com.google.dagger:dagger:2.21
 com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
+com.google.errorprone:error_prone_annotations:2.3.4
 com.google.errorprone:error_prone_check_api:2.3.3
 com.google.errorprone:error_prone_core:2.3.3
 com.google.errorprone:error_prone_type_annotations:2.3.3
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:27.0.1-jre
+com.google.guava:guava:28.2-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.j2objc:j2objc-annotations:1.1
+com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-com.squareup:javapoet:1.11.1
+com.squareup:javapoet:1.12.1
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
+javax.persistence:javax.persistence-api:2.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:2.5.3
+org.checkerframework:checker-qual:2.10.0
 org.checkerframework:dataflow:2.5.3
 org.checkerframework:javacutil:2.5.3
-org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2

core/gradle/dependency-locks/testCompile.lockfile

@@ -243,13 +243,16 @@ org.jboss:jandex:2.0.5.Final
 org.jetbrains:annotations:17.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.jupiter:junit-jupiter-migrationsupport:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.jupiter:junit-jupiter-migrationsupport:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.platform:junit-platform-launcher:1.6.1
+org.junit.platform:junit-platform-runner:1.6.1
+org.junit.platform:junit-platform-suite-api:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.jvnet.staxex:stax-ex:1.8
 org.mockito:mockito-core:2.25.0
 org.mortbay.jetty:jetty-util:6.1.26

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -241,13 +241,16 @@ org.jboss:jandex:2.0.5.Final
 org.jetbrains:annotations:17.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.jupiter:junit-jupiter-migrationsupport:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.jupiter:junit-jupiter-migrationsupport:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.platform:junit-platform-launcher:1.6.1
+org.junit.platform:junit-platform-runner:1.6.1
+org.junit.platform:junit-platform-suite-api:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.jvnet.staxex:stax-ex:1.8
 org.mockito:mockito-core:2.25.0
 org.mortbay.jetty:jetty-util:6.1.26

core/gradle/dependency-locks/testRuntime.lockfile

@@ -246,13 +246,16 @@ org.jboss:jandex:2.0.5.Final
 org.jetbrains:annotations:17.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.jupiter:junit-jupiter-migrationsupport:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.jupiter:junit-jupiter-migrationsupport:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.platform:junit-platform-launcher:1.6.1
+org.junit.platform:junit-platform-runner:1.6.1
+org.junit.platform:junit-platform-suite-api:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.jvnet.staxex:stax-ex:1.8
 org.mockito:mockito-core:2.25.0
 org.mortbay.jetty:jetty-util:6.1.26

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -246,13 +246,16 @@ org.jboss:jandex:2.0.5.Final
 org.jetbrains:annotations:17.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
-org.junit.jupiter:junit-jupiter-api:5.6.0
-org.junit.jupiter:junit-jupiter-engine:5.6.0
-org.junit.jupiter:junit-jupiter-migrationsupport:5.6.0
-org.junit.platform:junit-platform-commons:1.6.0
-org.junit.platform:junit-platform-engine:1.6.0
-org.junit.vintage:junit-vintage-engine:5.6.0
-org.junit:junit-bom:5.6.0
+org.junit.jupiter:junit-jupiter-api:5.6.1
+org.junit.jupiter:junit-jupiter-engine:5.6.1
+org.junit.jupiter:junit-jupiter-migrationsupport:5.6.1
+org.junit.platform:junit-platform-commons:1.6.1
+org.junit.platform:junit-platform-engine:1.6.1
+org.junit.platform:junit-platform-launcher:1.6.1
+org.junit.platform:junit-platform-runner:1.6.1
+org.junit.platform:junit-platform-suite-api:1.6.1
+org.junit.vintage:junit-vintage-engine:5.6.1
+org.junit:junit-bom:5.6.1
 org.jvnet.staxex:stax-ex:1.8
 org.mockito:mockito-core:2.25.0
 org.mortbay.jetty:jetty-util:6.1.26
@@ -280,6 +283,7 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.28
+org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.12.1
 org.testcontainers:jdbc:1.12.1
 org.testcontainers:postgresql:1.12.1

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -31,6 +31,7 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
+import google.registry.schema.domain.RegistryLock;
 import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.Retrier;
 import javax.inject.Inject;
@@ -149,7 +150,7 @@ public final class AsyncTaskEnqueuer {
 
   /** Enqueues a task to asynchronously refresh DNS for a renamed host. */
   public void enqueueAsyncDnsRefresh(HostResource host, DateTime now) {
-    VKey<HostResource> hostKey = host.createKey();
+    VKey<HostResource> hostKey = host.createVKey();
     logger.atInfo().log("Enqueuing async DNS refresh for renamed host %s.", hostKey);
     addTaskToQueueWithRetry(
         asyncDnsRefreshPullQueue,
@@ -158,6 +159,26 @@ public final class AsyncTaskEnqueuer {
             .param(PARAM_REQUESTED_TIME, now.toString()));
   }
 
+  /**
+   * Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked.
+   *
+   * <p>Note: the relockDuration must be present on the lock object.
+   */
+  public void enqueueDomainRelock(RegistryLock lock) {
+    checkArgument(
+        lock.getRelockDuration().isPresent(),
+        "Lock with ID %s not configured for relock",
+        lock.getRevisionId());
+    addTaskToQueueWithRetry(
+        asyncActionsPushQueue,
+        TaskOptions.Builder.withUrl(RelockDomainAction.PATH)
+            .method(Method.POST)
+            .param(
+                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
+                String.valueOf(lock.getRevisionId()))
+            .countdownMillis(lock.getRelockDuration().get().getMillis()));
+  }
+
   /**
    * Adds a task to a queue with retrying, to avoid aborting the entire flow over a transient issue
    * enqueuing a task.

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -283,7 +283,9 @@ public class DeleteContactsAndHostsAction implements Runnable {
     /** Determine whether the target resource is a linked resource on the domain. */
     private boolean isLinked(DomainBase domain, Key<? extends EppResource> resourceKey) {
       if (resourceKey.getKind().equals(KIND_CONTACT)) {
-        return domain.getReferencedContacts().contains(resourceKey);
+        return domain
+            .getReferencedContacts()
+            .contains(VKey.createOfy(ContactResource.class, (Key<ContactResource>) resourceKey));
       } else if (resourceKey.getKind().equals(KIND_HOST)) {
         return domain
             .getNameservers()

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -155,89 +155,100 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       }
       int numBillingEventsSaved = 0;
       try {
-        numBillingEventsSaved = tm().transactNew(() -> {
-          ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder =
-              new ImmutableSet.Builder<>();
-          final Registry tld = Registry.get(getTldFromDomainName(recurring.getTargetId()));
+        numBillingEventsSaved =
+            tm().transactNew(
+                    () -> {
+                      ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder =
+                          new ImmutableSet.Builder<>();
+                      final Registry tld =
+                          Registry.get(getTldFromDomainName(recurring.getTargetId()));
 
-          // Determine the complete set of times at which this recurring event should occur
-          // (up to and including the runtime of the mapreduce).
-          Iterable<DateTime> eventTimes =
-              recurring.getRecurrenceTimeOfYear().getInstancesInRange(Range.closed(
-                  recurring.getEventTime(),
-                  earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
+                      // Determine the complete set of times at which this recurring event should
+                      // occur (up to and including the runtime of the mapreduce).
+                      Iterable<DateTime> eventTimes =
+                          recurring
+                              .getRecurrenceTimeOfYear()
+                              .getInstancesInRange(
+                                  Range.closed(
+                                      recurring.getEventTime(),
+                                      earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
 
-          // Convert these event times to billing times
-          final ImmutableSet<DateTime> billingTimes =
-              getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
+                      // Convert these event times to billing times
+                      final ImmutableSet<DateTime> billingTimes =
+                          getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
 
-          Key<? extends EppResource> domainKey = recurring.getParentKey().getParent();
-          Iterable<OneTime> oneTimesForDomain =
-              ofy().load().type(OneTime.class).ancestor(domainKey);
+                      Key<? extends EppResource> domainKey = recurring.getParentKey().getParent();
+                      Iterable<OneTime> oneTimesForDomain =
+                          ofy().load().type(OneTime.class).ancestor(domainKey);
 
-          // Determine the billing times that already have OneTime events persisted.
-          ImmutableSet<DateTime> existingBillingTimes =
-              getExistingBillingTimes(oneTimesForDomain, recurring);
+                      // Determine the billing times that already have OneTime events persisted.
+                      ImmutableSet<DateTime> existingBillingTimes =
+                          getExistingBillingTimes(oneTimesForDomain, recurring);
 
-          ImmutableSet.Builder<HistoryEntry> historyEntriesBuilder =
-              new ImmutableSet.Builder<>();
-          // Create synthetic OneTime events for all billing times that do not yet have an event
-          // persisted.
-          for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
-            // Construct a new HistoryEntry that parents over the OneTime
-            HistoryEntry historyEntry = new HistoryEntry.Builder()
-                .setBySuperuser(false)
-                .setClientId(recurring.getClientId())
-                .setModificationTime(tm().getTransactionTime())
-                .setParent(domainKey)
-                .setPeriod(Period.create(1, YEARS))
-                .setReason("Domain autorenewal by ExpandRecurringBillingEventsAction")
-                .setRequestedByRegistrar(false)
-                .setType(DOMAIN_AUTORENEW)
-                // Don't write a domain transaction record if the recurrence was ended prior to the
-                // billing time (i.e. a domain was deleted during the autorenew grace period).
-                .setDomainTransactionRecords(
-                    recurring.getRecurrenceEndTime().isBefore(billingTime)
-                        ? ImmutableSet.of()
-                        : ImmutableSet.of(
-                            DomainTransactionRecord.create(
-                                tld.getTldStr(),
-                                // We report this when the autorenew grace period ends
-                                billingTime,
-                                TransactionReportField.netRenewsFieldFromYears(1),
-                                1)))
-                .build();
-            historyEntriesBuilder.add(historyEntry);
+                      ImmutableSet.Builder<HistoryEntry> historyEntriesBuilder =
+                          new ImmutableSet.Builder<>();
+                      // Create synthetic OneTime events for all billing times that do not yet have
+                      // an event persisted.
+                      for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
+                        // Construct a new HistoryEntry that parents over the OneTime
+                        HistoryEntry historyEntry =
+                            new HistoryEntry.Builder()
+                                .setBySuperuser(false)
+                                .setClientId(recurring.getClientId())
+                                .setModificationTime(tm().getTransactionTime())
+                                .setParent(domainKey)
+                                .setPeriod(Period.create(1, YEARS))
+                                .setReason(
+                                    "Domain autorenewal by ExpandRecurringBillingEventsAction")
+                                .setRequestedByRegistrar(false)
+                                .setType(DOMAIN_AUTORENEW)
+                                // Don't write a domain transaction record if the recurrence was
+                                // ended prior to the billing time (i.e. a domain was deleted
+                                // during the autorenew grace period).
+                                .setDomainTransactionRecords(
+                                    recurring.getRecurrenceEndTime().isBefore(billingTime)
+                                        ? ImmutableSet.of()
+                                        : ImmutableSet.of(
+                                            DomainTransactionRecord.create(
+                                                tld.getTldStr(),
+                                                // We report this when the autorenew grace period
+                                                // ends
+                                                billingTime,
+                                                TransactionReportField.netRenewsFieldFromYears(1),
+                                                1)))
+                                .build();
+                        historyEntriesBuilder.add(historyEntry);
 
-            DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
-            // Determine the cost for a one-year renewal.
-            Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
-            syntheticOneTimesBuilder.add(new OneTime.Builder()
-                .setBillingTime(billingTime)
-                .setClientId(recurring.getClientId())
-                .setCost(renewCost)
-                .setEventTime(eventTime)
-                .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
-                .setParent(historyEntry)
-                .setPeriodYears(1)
-                .setReason(recurring.getReason())
-                .setSyntheticCreationTime(executeTime)
-                .setCancellationMatchingBillingEvent(Key.create(recurring))
-                .setTargetId(recurring.getTargetId())
-                .build());
-          }
-          Set<HistoryEntry> historyEntries = historyEntriesBuilder.build();
-          Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
-          if (!isDryRun) {
-            ImmutableSet<ImmutableObject> entitiesToSave =
-                new ImmutableSet.Builder<ImmutableObject>()
-                    .addAll(historyEntries)
-                    .addAll(syntheticOneTimes)
-                    .build();
-            ofy().save().entities(entitiesToSave).now();
-          }
-          return syntheticOneTimes.size();
-        });
+                        DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
+                        // Determine the cost for a one-year renewal.
+                        Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
+                        syntheticOneTimesBuilder.add(
+                            new OneTime.Builder()
+                                .setBillingTime(billingTime)
+                                .setClientId(recurring.getClientId())
+                                .setCost(renewCost)
+                                .setEventTime(eventTime)
+                                .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
+                                .setParent(historyEntry)
+                                .setPeriodYears(1)
+                                .setReason(recurring.getReason())
+                                .setSyntheticCreationTime(executeTime)
+                                .setCancellationMatchingBillingEvent(recurring.createVKey())
+                                .setTargetId(recurring.getTargetId())
+                                .build());
+                      }
+                      Set<HistoryEntry> historyEntries = historyEntriesBuilder.build();
+                      Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
+                      if (!isDryRun) {
+                        ImmutableSet<ImmutableObject> entitiesToSave =
+                            new ImmutableSet.Builder<ImmutableObject>()
+                                .addAll(historyEntries)
+                                .addAll(syntheticOneTimes)
+                                .build();
+                        ofy().save().entities(entitiesToSave).now();
+                      }
+                      return syntheticOneTimes.size();
+                    });
       } catch (Throwable t) {
         getContext().incrementCounter("error: " + t.getClass().getSimpleName());
         getContext().incrementCounter(ERROR_COUNTER);
@@ -279,7 +290,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       return Streams.stream(oneTimesForDomain)
           .filter(
               billingEvent ->
-                  Key.create(recurringEvent)
+                  recurringEvent
+                      .createVKey()
                       .equals(billingEvent.getCancellationMatchingBillingEvent()))
           .map(OneTime::getBillingTime)
           .collect(toImmutableSet());

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -50,6 +50,7 @@ import javax.inject.Inject;
 public class RelockDomainAction implements Runnable {
 
   public static final String PATH = "/_dr/task/relockDomain";
+  public static final String OLD_UNLOCK_REVISION_ID_PARAM = "oldUnlockRevisionId";
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -59,7 +60,7 @@ public class RelockDomainAction implements Runnable {
 
   @Inject
   public RelockDomainAction(
-      @Parameter("oldUnlockRevisionId") long oldUnlockRevisionId,
+      @Parameter(OLD_UNLOCK_REVISION_ID_PARAM) long oldUnlockRevisionId,
       DomainLockUtils domainLockUtils,
       Response response) {
     this.oldUnlockRevisionId = oldUnlockRevisionId;

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -0,0 +1,86 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a
+ * Datastore export and Nomulus CommitLogs that overlap with the export.
+ */
+public final class BackupPaths {
+
+  private BackupPaths() {}
+
+  private static final String WILDCARD_CHAR = "*";
+  private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/input-%s";
+  /**
+   * Regex pattern that captures the kind string in a file name. Datastore places no restrictions on
+   * what characters may be used in a kind string.
+   */
+  private static final String FILENAME_TO_KIND_REGEX = ".+/all_namespaces/kind_(.+)/input-.+";
+
+  private static final Pattern FILENAME_TO_KIND_PATTERN = Pattern.compile(FILENAME_TO_KIND_REGEX);
+
+  /**
+   * Returns a regex pattern that matches all Datastore export files of a given {@code kind}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kind the 'kind' of the Datastore entity
+   */
+  public static String getExportFileNamePattern(String exportDir, String kind) {
+    checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
+    checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, WILDCARD_CHAR);
+  }
+
+  /**
+   * Returns the fully qualified path of a Datastore export file with the given {@code kind} and
+   * {@code shard}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kind the 'kind' of the Datastore entity
+   * @param shard an integer suffix of the file name
+   */
+  public static String getExportFileNameByShard(String exportDir, String kind, int shard) {
+    checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
+    checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
+    checkArgument(shard >= 0, "Negative shard %s not allowed.", shard);
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, Integer.toString(shard));
+  }
+
+  /**
+   * Returns the 'kind' of entity stored in a file based on the file name.
+   *
+   * <p>This method poses low risk and greatly simplifies the implementation of some transforms in
+   * {@link ExportLoadingTransforms}.
+   *
+   * @see ExportLoadingTransforms
+   */
+  public static String getKindFromFileName(String fileName) {
+    checkArgument(!isNullOrEmpty(fileName), "Null or empty fileName.");
+    Matcher matcher = FILENAME_TO_KIND_PATTERN.matcher(fileName);
+    checkArgument(
+        matcher.matches(),
+        "Illegal file name %s, should match %s.",
+        fileName,
+        FILENAME_TO_KIND_REGEX);
+    return matcher.group(1);
+  }
+}

core/src/main/java/google/registry/beam/initsql/ExportLoadingTransforms.java

@@ -0,0 +1,117 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static google.registry.beam.initsql.BackupPaths.getExportFileNamePattern;
+import static google.registry.beam.initsql.BackupPaths.getKindFromFileName;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.collect.ImmutableList;
+import google.registry.tools.LevelDbLogReader;
+import java.io.IOException;
+import java.util.Collection;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+/**
+ * {@link PTransform Pipeline transforms} for loading from Datastore export files. They are all part
+ * of a transformation that loads raw records from a Datastore export, and are broken apart for
+ * testing.
+ *
+ * <p>We drop the 'kind' information in {@link #getDatastoreExportFilePatterns} and recover it later
+ * using the file paths. Although we could have kept it by passing around {@link KV key-value
+ * pairs}, the code would be more complicated, especially in {@link #loadDataFromFiles()}.
+ */
+public class ExportLoadingTransforms {
+
+  /**
+   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
+   * all Datastore export files of the given {@code kinds}.
+   */
+  public static PTransform<PBegin, PCollection<String>> getDatastoreExportFilePatterns(
+      String exportDir, Collection<String> kinds) {
+    return Create.of(
+            kinds.stream()
+                .map(kind -> getExportFileNamePattern(exportDir, kind))
+                .collect(ImmutableList.toImmutableList()))
+        .withCoder(StringUtf8Coder.of());
+  }
+
+  /**
+   * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}.
+   */
+  public static PTransform<PCollection<String>, PCollection<Metadata>> getFilesByPatterns() {
+    return new PTransform<PCollection<String>, PCollection<Metadata>>() {
+      @Override
+      public PCollection<Metadata> expand(PCollection<String> input) {
+        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+      }
+    };
+  }
+
+  /**
+   * Returns a {@link PTransform} from file {@link Metadata} to {@link KV Key-Value pairs} with
+   * entity 'kind' as key and raw record as value.
+   */
+  public static PTransform<PCollection<Metadata>, PCollection<KV<String, byte[]>>>
+      loadDataFromFiles() {
+    return new PTransform<PCollection<Metadata>, PCollection<KV<String, byte[]>>>() {
+      @Override
+      public PCollection<KV<String, byte[]>> expand(PCollection<Metadata> input) {
+        return input
+            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+            .apply(
+                MapElements.into(kvs(strings(), TypeDescriptor.of(ReadableFile.class)))
+                    .via(file -> KV.of(getKindFromFileName(file.getMetadata().toString()), file)))
+            .apply("Load One LevelDb File", ParDo.of(new LoadOneFile()));
+      }
+    };
+  }
+
+  /**
+   * Reads a LevelDb file and converts each raw record into a {@link KV pair} of kind and bytes.
+   *
+   * <p>LevelDb files are not seekable because a large object may span multiple blocks. If a
+   * sequential read fails, the file needs to be retried from the beginning.
+   */
+  private static class LoadOneFile extends DoFn<KV<String, ReadableFile>, KV<String, byte[]>> {
+
+    @ProcessElement
+    public void processElement(
+        @Element KV<String, ReadableFile> kv, OutputReceiver<KV<String, byte[]>> output) {
+      try {
+        LevelDbLogReader.from(kv.getValue().open())
+            .forEachRemaining(record -> output.output(KV.of(kv.getKey(), record)));
+      } catch (IOException e) {
+        // Let the pipeline retry the whole file.
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

core/src/main/java/google/registry/beam/initsql/README.md

@@ -0,0 +1,3 @@
+## Summary
+
+This package contains a BEAM pipeline that populates a Cloud SQL database from a Datastore backup.

core/src/main/java/google/registry/beam/spec11/ThreatMatch.java

@@ -24,22 +24,10 @@ import org.json.JSONObject;
 public abstract class ThreatMatch implements Serializable {
 
   private static final String THREAT_TYPE_FIELD = "threatType";
-  private static final String PLATFORM_TYPE_FIELD = "platformType";
-  private static final String METADATA_FIELD = "threatEntryMetadata";
   private static final String DOMAIN_NAME_FIELD = "fullyQualifiedDomainName";
 
   /** Returns what kind of threat it is (malware, phishing etc.) */
   public abstract String threatType();
-  /** Returns what platforms it affects (Windows, Linux etc.) */
-  abstract String platformType();
-  /**
-   * Returns a String representing a JSON Object containing arbitrary metadata associated with this
-   * threat, or "NONE" if there is no metadata to retrieve.
-   *
-   * <p>This ideally would be a {@link JSONObject} type, but can't be due to serialization
-   * requirements.
-   */
-  abstract String metadata();
   /** Returns the fully qualified domain name [SLD].[TLD] of the matched threat. */
   public abstract String fullyQualifiedDomainName();
 
@@ -52,29 +40,19 @@ public abstract class ThreatMatch implements Serializable {
   static ThreatMatch create(JSONObject threatMatchJSON, String fullyQualifiedDomainName)
       throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatchJSON.getString(THREAT_TYPE_FIELD),
-        threatMatchJSON.getString(PLATFORM_TYPE_FIELD),
-        threatMatchJSON.has(METADATA_FIELD)
-            ? threatMatchJSON.getJSONObject(METADATA_FIELD).toString()
-            : "NONE",
-        fullyQualifiedDomainName);
+        threatMatchJSON.getString(THREAT_TYPE_FIELD), fullyQualifiedDomainName);
   }
 
   /** Returns a {@link JSONObject} representing a subset of this object's data. */
   JSONObject toJSON() throws JSONException {
     return new JSONObject()
         .put(THREAT_TYPE_FIELD, threatType())
-        .put(PLATFORM_TYPE_FIELD, platformType())
-        .put(METADATA_FIELD, metadata())
         .put(DOMAIN_NAME_FIELD, fullyQualifiedDomainName());
   }
 
   /** Parses a {@link JSONObject} and returns an equivalent {@link ThreatMatch}. */
   public static ThreatMatch fromJSON(JSONObject threatMatch) throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatch.getString(THREAT_TYPE_FIELD),
-        threatMatch.getString(PLATFORM_TYPE_FIELD),
-        threatMatch.getString(METADATA_FIELD),
-        threatMatch.getString(DOMAIN_NAME_FIELD));
+        threatMatch.getString(THREAT_TYPE_FIELD), threatMatch.getString(DOMAIN_NAME_FIELD));
   }
 }

core/src/main/java/google/registry/bigquery/BigqueryConnection.java

@@ -442,10 +442,8 @@ public class BigqueryConnection implements AutoCloseable {
    * Returns the result of calling queryToLocalTable, but synchronously to avoid spawning new
    * background threads, which App Engine doesn't support.
    *
-   * @see <a href="https://cloud.google.com/appengine/docs/standard/java/runtime#Threads">
-   *   App Engine Runtime</a>
-   *
-   * <p>Returns the results of the query in an ImmutableTable on success.
+   * @see <a href="https://cloud.google.com/appengine/docs/standard/java/runtime#Threads">App Engine
+   *     Runtime</a>
    */
   public ImmutableTable<Integer, TableFieldSchema, Object> queryToLocalTableSync(String querySql) {
     Job job = new Job()
@@ -576,8 +574,6 @@ public class BigqueryConnection implements AutoCloseable {
 
   /**
    * Launch a job, wait for it to complete, but <i>do not</i> check for errors.
-   *
-   * @throws BigqueryJobFailureException
    */
   public Job runJob(Job job, @Nullable AbstractInputStreamContent data) {
     return checkJob(waitForJob(launchJob(job, data)));

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -207,8 +207,20 @@ hibernate:
 
   # Connection pool configurations.
   hikariConnectionTimeout: 20000
-  hikariMinimumIdle: 0
-  hikariMaximumPoolSize: 20
+  # We occasionally received "Connection is not available, request timed out"
+  # exception when setting minimumIdle to 0 and it turned out it is a bug (See
+  # https://github.com/brettwooldridge/HikariCP/issues/1212) in HikariCP.
+  #
+  # We tried to use a fixed size pool but ran into an issue(See b/155383029),
+  # so we need further investigation to figure out the proper size of the pool.
+  #
+  # HikariCP also recommends not setting minimumIdle for maximum performance
+  # and responsiveness to spike demands (See
+  # https://github.com/brettwooldridge/HikariCP).
+  #
+  # TODO(b/154720215): Investigate the long term fix.
+  hikariMinimumIdle: 1
+  hikariMaximumPoolSize: 10
   hikariIdleTimeout: 300000
 
 cloudSql:

core/src/main/java/google/registry/dns/writer/dnsupdate/DnsMessageTransport.java

@@ -84,7 +84,7 @@ public class DnsMessageTransport {
    * @param query a message to send
    * @return the response received from the server
    * @throws IOException if the Socket input/output streams throws one
-   * @throws IllegalArgumentException if the query is too large to be sent (> 65535 bytes)
+   * @throws IllegalArgumentException if the query is too large to be sent (> 65535 bytes)
    */
   public Message send(Message query) throws IOException {
     try (Socket socket = factory.createSocket(InetAddress.getByName(updateHost), DNS_PORT)) {

core/src/main/java/google/registry/export/BigqueryPollJobAction.java

@@ -73,8 +73,10 @@ public class BigqueryPollJobAction implements Runnable {
 
   @Override
   public void run() {
-    checkJobOutcome();  // Throws a NotModifiedException if the job hasn't completed.
-    if (payload == null || payload.length == 0) {
+    boolean jobOutcome =
+        checkJobOutcome(); // Throws a NotModifiedException if the job hasn't completed.
+    // If the job failed, do not enqueue the next step.
+    if (!jobOutcome || payload == null || payload.length == 0) {
       return;
     }
     // If there is a payload, it's a chained task, so enqueue it.

core/src/main/java/google/registry/flows/EppXmlSanitizer.java

@@ -42,8 +42,8 @@ import javax.xml.stream.events.XMLEvent;
 /**
  * Sanitizes sensitive data in incoming/outgoing EPP XML messages.
  *
- * <p>Current implementation masks user credentials (text following <pw> and <newPW> tags) as
- * follows:
+ * <p>Current implementation masks user credentials (text following &lt;pw&gt; and &lt;newPW&gt;
+ * tags) as follows:
  *
  * <ul>
  *   <li>A control character (in ranges [0 - 1F] and [7F - 9F]) is replaced with 'C'.

core/src/main/java/google/registry/flows/ResourceFlowUtils.java

@@ -48,6 +48,7 @@ import google.registry.model.eppcommon.AuthInfo;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -93,11 +94,12 @@ public final class ResourceFlowUtils {
                    * trust the query and need to do the full mapreduce.
                    */
                   Iterable<Key<DomainBase>> keys =
-                      queryForLinkedDomains(fki.getResourceKey(), now)
+                      queryForLinkedDomains(fki.getResourceKey().getOfyKey(), now)
                           .limit(FAILFAST_CHECK_COUNT)
                           .keys();
+                  VKey<R> resourceVKey = fki.getResourceKey();
                   Predicate<DomainBase> predicate =
-                      domain -> getPotentialReferences.apply(domain).contains(fki.getResourceKey());
+                      domain -> getPotentialReferences.apply(domain).contains(resourceVKey);
                   return ofy().load().keys(keys).values().stream().anyMatch(predicate)
                       ? new ResourceToDeleteIsReferencedException()
                       : null;
@@ -134,9 +136,9 @@ public final class ResourceFlowUtils {
 
   public static <R extends EppResource> void verifyResourceDoesNotExist(
       Class<R> clazz, String targetId, DateTime now, String clientId) throws EppException {
-    Key<R> key = loadAndGetKey(clazz, targetId, now);
+    VKey<R> key = loadAndGetKey(clazz, targetId, now);
     if (key != null) {
-      R resource = ofy().load().key(key).now();
+      R resource = tm().load(key);
       // These are similar exceptions, but we can track them internally as log-based metrics.
       if (Objects.equals(clientId, resource.getPersistedCurrentSponsorClientId())) {
         throw new ResourceAlreadyExistsForThisClientException(targetId);
@@ -184,9 +186,8 @@ public final class ResourceFlowUtils {
     }
     // The roid should match one of the contacts.
     Optional<Key<ContactResource>> foundContact =
-        domain
-            .getReferencedContacts()
-            .stream()
+        domain.getReferencedContacts().stream()
+            .map(VKey::getOfyKey)
             .filter(key -> key.getName().equals(authRepoId))
             .findFirst();
     if (!foundContact.isPresent()) {

core/src/main/java/google/registry/flows/custom/DomainCheckFlowCustomLogic.java

@@ -75,7 +75,7 @@ public class DomainCheckFlowCustomLogic extends BaseFlowCustomLogic {
 
     /**
      * The time to perform the domain check as of. This defaults to the current time, but can be
-     * overridden in v>=0.12 of the fee extension.
+     * overridden in v>=0.12 of the fee extension.
      */
     public abstract DateTime asOfDate();
 
@@ -105,7 +105,7 @@ public class DomainCheckFlowCustomLogic extends BaseFlowCustomLogic {
 
     /**
      * The time to perform the domain check as of. This defaults to the current time, but can be
-     * overridden in v>=0.12 of the fee extension.
+     * overridden in v>=0.12 of the fee extension.
      */
     public abstract DateTime asOfDate();
 

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -357,9 +357,7 @@ public class DomainCreateFlow implements TransactionalFlow {
             .setFullyQualifiedDomainName(targetId)
             .setNameservers(
                 (ImmutableSet<VKey<HostResource>>)
-                    command.getNameservers().stream()
-                        .map(key -> VKey.createOfy(HostResource.class, key))
-                        .collect(toImmutableSet()))
+                    command.getNameservers().stream().collect(toImmutableSet()))
             .setStatusValues(statuses.build())
             .setContacts(command.getContacts())
             .addGracePeriod(GracePeriod.forBillingEvent(GracePeriodStatus.ADD, createBillingEvent))
@@ -533,7 +531,7 @@ public class DomainCreateFlow implements TransactionalFlow {
         .setPeriodYears(years)
         .setCost(feesAndCredits.getCreateCost())
         .setEventTime(now)
-        .setAllocationToken(allocationToken.map(Key::create).orElse(null))
+        .setAllocationToken(allocationToken.map(AllocationToken::createVKey).orElse(null))
         .setBillingTime(
             now.plus(
                 isAnchorTenant

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -212,6 +212,28 @@ public final class DomainDeleteFlow implements TransactionalFlow {
       builder.setDeletePollMessage(Key.create(deletePollMessage));
     }
 
+    // Cancel any grace periods that were still active, and set the expiration time accordingly.
+    DateTime newExpirationTime = existingDomain.getRegistrationExpirationTime();
+    for (GracePeriod gracePeriod : existingDomain.getGracePeriods()) {
+      // No cancellation is written if the grace period was not for a billable event.
+      if (gracePeriod.hasBillingEvent()) {
+        entitiesToSave.add(
+            BillingEvent.Cancellation.forGracePeriod(gracePeriod, historyEntry, targetId));
+        if (gracePeriod.getOneTimeBillingEvent() != null) {
+          // Take the amount of amount of registration time being refunded off the expiration time.
+          // This can be either add grace periods or renew grace periods.
+          BillingEvent.OneTime oneTime =
+              ofy().load().key(gracePeriod.getOneTimeBillingEvent()).now();
+          newExpirationTime = newExpirationTime.minusYears(oneTime.getPeriodYears());
+        } else if (gracePeriod.getRecurringBillingEvent() != null) {
+          // Take 1 year off the registration if in the autorenew grace period (no need to load the
+          // recurring billing event; all autorenews are for 1 year).
+          newExpirationTime = newExpirationTime.minusYears(1);
+        }
+      }
+    }
+    builder.setRegistrationExpirationTime(newExpirationTime);
+
     DomainBase newDomain = builder.build();
     updateForeignKeyIndexDeletionTime(newDomain);
     handlePendingTransferOnDelete(existingDomain, newDomain, now, historyEntry);
@@ -221,14 +243,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
     // event and poll message will already have been deleted in
     // ResourceDeleteFlow since it's listed in serverApproveEntities.
     dnsQueue.addDomainRefreshTask(existingDomain.getFullyQualifiedDomainName());
-    // Cancel any grace periods that were still active.
-    for (GracePeriod gracePeriod : existingDomain.getGracePeriods()) {
-      // No cancellation is written if the grace period was not for a billable event.
-      if (gracePeriod.hasBillingEvent()) {
-        entitiesToSave.add(
-            BillingEvent.Cancellation.forGracePeriod(gracePeriod, historyEntry, targetId));
-      }
-    }
+
     entitiesToSave.add(newDomain, historyEntry);
     EntityChanges entityChanges = flowCustomLogic.beforeSave(
         BeforeSaveParameters.newBuilder()

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -35,6 +35,7 @@ import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRIS
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_ANCHOR_TENANT;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_SPECIFIC_USE;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.pricing.PricingEngineProxy.isDomainPremium;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
@@ -120,6 +121,7 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tmch.ClaimsListShard;
+import google.registry.persistence.VKey;
 import google.registry.tldconfig.idn.IdnLabelValidator;
 import google.registry.util.Idn;
 import java.math.BigDecimal;
@@ -305,10 +307,10 @@ public class DomainFlowUtils {
   /** Verify that no linked resources have disallowed statuses. */
   static void verifyNotInPendingDelete(
       Set<DesignatedContact> contacts,
-      Key<ContactResource> registrant,
-      Set<Key<HostResource>> nameservers)
+      VKey<ContactResource> registrant,
+      Set<VKey<HostResource>> nameservers)
       throws EppException {
-    ImmutableList.Builder<Key<? extends EppResource>> keysToLoad = new ImmutableList.Builder<>();
+    ImmutableList.Builder<VKey<? extends EppResource>> keysToLoad = new ImmutableList.Builder<>();
     contacts.stream().map(DesignatedContact::getContactKey).forEach(keysToLoad::add);
     Optional.ofNullable(registrant).ifPresent(keysToLoad::add);
     keysToLoad.addAll(nameservers);
@@ -355,7 +357,7 @@ public class DomainFlowUtils {
         contacts.stream()
             .collect(
                 toImmutableSetMultimap(
-                    DesignatedContact::getType, DesignatedContact::getContactKey));
+                    DesignatedContact::getType, contact -> contact.getContactKey().getOfyKey()));
 
     // If any contact type has multiple contacts:
     if (contactsByType.asMap().values().stream().anyMatch(v -> v.size() > 1)) {
@@ -376,7 +378,7 @@ public class DomainFlowUtils {
   }
 
   static void validateRequiredContactsPresent(
-      @Nullable Key<ContactResource> registrant, Set<DesignatedContact> contacts)
+      @Nullable VKey<ContactResource> registrant, Set<DesignatedContact> contacts)
       throws RequiredParameterMissingException {
     if (registrant == null) {
       throw new MissingRegistrantException();
@@ -978,7 +980,7 @@ public class DomainFlowUtils {
     for (DesignatedContact contact : contacts) {
       builder.add(
           ForeignKeyedDesignatedContact.create(
-              contact.getType(), ofy().load().key(contact.getContactKey()).now().getContactId()));
+              contact.getType(), tm().load(contact.getContactKey()).getContactId()));
     }
     return builder.build();
   }

core/src/main/java/google/registry/flows/domain/DomainInfoFlow.java

@@ -21,7 +21,6 @@ import static google.registry.flows.domain.DomainFlowUtils.addSecDnsExtensionIfP
 import static google.registry.flows.domain.DomainFlowUtils.handleFeeRequest;
 import static google.registry.flows.domain.DomainFlowUtils.loadForeignKeyedDesignatedContacts;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
@@ -102,16 +101,16 @@ public final class DomainInfoFlow implements Flow {
     flowCustomLogic.afterValidation(
         AfterValidationParameters.newBuilder().setDomain(domain).build());
     // Prefetch all referenced resources. Calling values() blocks until loading is done.
-    // We do nameservers separately since they've been converted to VKey.
     tm().load(domain.getNameservers());
-    ofy().load().values(domain.getReferencedContacts()).values();
+    tm().load(domain.getReferencedContacts());
     // Registrars can only see a few fields on unauthorized domains.
     // This is a policy decision that is left up to us by the rfcs.
-    DomainInfoData.Builder infoBuilder = DomainInfoData.newBuilder()
-        .setFullyQualifiedDomainName(domain.getFullyQualifiedDomainName())
-        .setRepoId(domain.getRepoId())
-        .setCurrentSponsorClientId(domain.getCurrentSponsorClientId())
-        .setRegistrant(ofy().load().key(domain.getRegistrant()).now().getContactId());
+    DomainInfoData.Builder infoBuilder =
+        DomainInfoData.newBuilder()
+            .setFullyQualifiedDomainName(domain.getFullyQualifiedDomainName())
+            .setRepoId(domain.getRepoId())
+            .setCurrentSponsorClientId(domain.getCurrentSponsorClientId())
+            .setRegistrant(tm().load(domain.getRegistrant()).getContactId());
     // If authInfo is non-null, then the caller is authorized to see the full information since we
     // will have already verified the authInfo is valid.
     if (clientId.equals(domain.getCurrentSponsorClientId()) || authInfo.isPresent()) {

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -72,11 +72,9 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
-import google.registry.model.host.HostResource;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.persistence.VKey;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
@@ -246,14 +244,8 @@ public final class DomainUpdateFlow implements TransactionalFlow {
             .setLastEppUpdateClientId(clientId)
             .addStatusValues(add.getStatusValues())
             .removeStatusValues(remove.getStatusValues())
-            .addNameservers(
-                add.getNameservers().stream()
-                    .map(key -> VKey.createOfy(HostResource.class, key))
-                    .collect(toImmutableSet()))
-            .removeNameservers(
-                remove.getNameservers().stream()
-                    .map(key -> VKey.createOfy(HostResource.class, key))
-                    .collect(toImmutableSet()))
+            .addNameservers(add.getNameservers().stream().collect(toImmutableSet()))
+            .removeNameservers(remove.getNameservers().stream().collect(toImmutableSet()))
             .addContacts(add.getContacts())
             .removeContacts(remove.getContacts())
             .setRegistrant(firstNonNull(change.getRegistrant(), domain.getRegistrant()))

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -14,7 +14,6 @@
 
 package google.registry.flows.host;
 
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.ResourceFlowUtils.failfastForAsyncDelete;
 import static google.registry.flows.ResourceFlowUtils.loadAndVerifyExistence;
@@ -82,17 +81,6 @@ public final class HostDeleteFlow implements TransactionalFlow {
   @Inject EppResponse.Builder responseBuilder;
   @Inject HostDeleteFlow() {}
 
-  /**
-   * Hack to convert DomainBase's nameserver VKey's to Ofy Key's.
-   *
-   * <p>We currently need this because {@code failfastForAsyncDelete()} checks to see if a name is
-   * in the ofy keys and is used for both nameservers and contacts. When we convert contacts to
-   * VKey's, we can remove this and do the conversion in {@code failfastForAsyncDelete()}.
-   */
-  private static ImmutableSet<Key<HostResource>> getNameserverOfyKeys(DomainBase domain) {
-    return domain.getNameservers().stream().map(key -> key.getOfyKey()).collect(toImmutableSet());
-  }
-
   @Override
   public final EppResponse run() throws EppException {
     extensionManager.register(MetadataExtension.class);
@@ -100,7 +88,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
     validateClientIsLoggedIn(clientId);
     DateTime now = tm().getTransactionTime();
     validateHostName(targetId);
-    failfastForAsyncDelete(targetId, now, HostResource.class, HostDeleteFlow::getNameserverOfyKeys);
+    failfastForAsyncDelete(targetId, now, HostResource.class, DomainBase::getNameservers);
     HostResource existingHost = loadAndVerifyExistence(HostResource.class, targetId, now);
     verifyNoDisallowedStatuses(existingHost, DISALLOWED_STATUSES);
     if (!isSuperuser) {

core/src/main/java/google/registry/model/EppResource.java

@@ -16,12 +16,11 @@ package google.registry.model;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
@@ -32,11 +31,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Index;
@@ -44,12 +41,13 @@ import google.registry.config.RegistryConfig;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.transfer.TransferData;
+import google.registry.persistence.VKey;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.StreamSupport;
 import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
@@ -70,9 +68,11 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /** The ID of the registrar that is currently sponsoring this resource. */
   @Index
+  @Column(nullable = false)
   String currentSponsorClientId;
 
   /** The ID of the registrar that created this resource. */
+  @Column(nullable = false)
   String creationClientId;
 
   /**
@@ -88,7 +88,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   // Map the method to XML, not the field, because if we map the field (with an adaptor class) it
   // will never be omitted from the xml even if the timestamp inside creationTime is null and we
   // return null from the adaptor. (Instead it gets written as an empty tag.)
-  @Index CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
+  @Column(nullable = false)
+  @Index
+  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.
@@ -131,7 +133,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   @Transient
   ImmutableSortedMap<DateTime, Key<CommitLogManifest>> revisions = ImmutableSortedMap.of();
 
-  public final String getRepoId() {
+  public String getRepoId() {
     return repoId;
   }
 
@@ -179,6 +181,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   /** Get the foreign key string for this resource. */
   public abstract String getForeignKey();
 
+  /** Create the VKey for the specified EPP resource. */
+  public abstract VKey<? extends EppResource> createVKey();
+
   /** Override of {@link Buildable#asBuilder} so that the extra methods are visible. */
   @Override
   public abstract Builder<?, ?> asBuilder();
@@ -325,18 +330,18 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     }
   }
 
-  static final CacheLoader<Key<? extends EppResource>, EppResource> CACHE_LOADER =
-      new CacheLoader<Key<? extends EppResource>, EppResource>() {
+  static final CacheLoader<VKey<? extends EppResource>, EppResource> CACHE_LOADER =
+      new CacheLoader<VKey<? extends EppResource>, EppResource>() {
 
         @Override
-        public EppResource load(Key<? extends EppResource> key) {
-          return tm().doTransactionless(() -> ofy().load().key(key).now());
+        public EppResource load(VKey<? extends EppResource> key) {
+          return tm().doTransactionless(() -> tm().load(key));
         }
 
         @Override
-        public Map<Key<? extends EppResource>, EppResource> loadAll(
-            Iterable<? extends Key<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> loadMultiple(keys));
+        public Map<VKey<? extends EppResource>, EppResource> loadAll(
+            Iterable<? extends VKey<? extends EppResource>> keys) {
+          return tm().doTransactionless(() -> loadAsMap(keys));
         }
       };
 
@@ -348,10 +353,10 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * directly should of course never use the cache.
    */
   @NonFinalForTesting
-  private static LoadingCache<Key<? extends EppResource>, EppResource> cacheEppResources =
+  private static LoadingCache<VKey<? extends EppResource>, EppResource> cacheEppResources =
       createEppResourcesCache(getEppResourceCachingDuration());
 
-  private static LoadingCache<Key<? extends EppResource>, EppResource> createEppResourcesCache(
+  private static LoadingCache<VKey<? extends EppResource>, EppResource> createEppResourcesCache(
       Duration expiry) {
     return CacheBuilder.newBuilder()
         .expireAfterWrite(expiry.getMillis(), MILLISECONDS)
@@ -365,29 +370,16 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     cacheEppResources = createEppResourcesCache(effectiveExpiry);
   }
 
-  private static ImmutableMap<Key<? extends EppResource>, EppResource> loadMultiple(
-      Iterable<? extends Key<? extends EppResource>> keys) {
-    // This cast is safe because, in Objectify, Key<? extends EppResource> can also be
-    // treated as a Key<EppResource>.
-    @SuppressWarnings("unchecked")
-    ImmutableList<Key<EppResource>> typedKeys =
-        Streams.stream(keys).map(key -> (Key<EppResource>) key).collect(toImmutableList());
-
-    // Typing shenanigans are required to return the map with the correct required generic type.
-    return ofy().load().keys(typedKeys).entrySet().stream()
-        .collect(ImmutableMap.toImmutableMap(Entry::getKey, Entry::getValue));
-  }
-
   /**
    * Loads the given EppResources by their keys using the cache (if enabled).
    *
    * <p>Don't use this unless you really need it for performance reasons, and be sure that you are
    * OK with the trade-offs in loss of transactional consistency.
    */
-  public static ImmutableMap<Key<? extends EppResource>, EppResource> loadCached(
-      Iterable<Key<? extends EppResource>> keys) {
+  public static ImmutableMap<VKey<? extends EppResource>, EppResource> loadCached(
+      Iterable<VKey<? extends EppResource>> keys) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return loadMultiple(keys);
+      return loadAsMap(keys);
     }
     try {
       return cacheEppResources.getAll(keys);
@@ -402,9 +394,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>Don't use this unless you really need it for performance reasons, and be sure that you are
    * OK with the trade-offs in loss of transactional consistency.
    */
-  public static <T extends EppResource> T loadCached(Key<T> key) {
+  public static <T extends EppResource> T loadCached(VKey<T> key) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return ofy().load().key(key).now();
+      return tm().load(key);
     }
     try {
       // Safe to cast because loading a Key<T> returns an entity of type T.
@@ -415,4 +407,17 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
       throw new RuntimeException("Error loading cached EppResources", e.getCause());
     }
   }
+
+  private static ImmutableMap<VKey<? extends EppResource>, EppResource> loadAsMap(
+      Iterable<? extends VKey<? extends EppResource>> keys) {
+    return StreamSupport.stream(keys.spliterator(), false)
+        // It's possible for us to receive the same key more than once which causes
+        // the immutable map build to break with a duplicate key, so we have to ensure key
+        // uniqueness.
+        .distinct()
+        // We have to use "key -> key" here instead of the identity() function, because
+        // the latter breaks the fairly complicated generic type checking required by the
+        // caching interface.
+        .collect(toImmutableMap(key -> key, key -> tm().load(key)));
+  }
 }

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -17,6 +17,7 @@ package google.registry.model;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;
@@ -141,7 +142,7 @@ public final class EppResourceUtils {
     T resource =
         useCache
             ? EppResource.loadCached(fki.getResourceKey())
-            : ofy().load().key(fki.getResourceKey()).now();
+            : tm().maybeLoad(fki.getResourceKey()).orElse(null);
     if (resource == null || isAtOrAfter(now, resource.getDeletionTime())) {
       return Optional.empty();
     }

core/src/main/java/google/registry/model/ModelUtils.java

@@ -34,6 +34,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Parent;
+import google.registry.persistence.VKey;
 import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
@@ -139,10 +140,13 @@ public class ModelUtils {
 
       // If the field's type is the same as the field's class object, then it's a non-parameterized
       // type, and thus we just add it directly. We also don't bother looking at the parameterized
-      // types of Key objects, since they are just references to other objects and don't actually
-      // embed themselves in the persisted object anyway.
+      // types of Key and VKey objects, since they are just references to other objects and don't
+      // actually embed themselves in the persisted object anyway.
       Class<?> fieldClazz = field.getType();
       Type fieldType = field.getGenericType();
+      if (VKey.class.equals(fieldClazz)) {
+        continue;
+      }
       builder.add(fieldClazz);
       if (fieldType.equals(fieldClazz) || Key.class.equals(clazz)) {
         continue;

core/src/main/java/google/registry/model/OteStats.java

@@ -50,7 +50,8 @@ import java.util.stream.Collectors;
 public class OteStats {
 
   /**
-   * Returns the statistics about the OT&E actions that have been taken by a particular registrar.
+   * Returns the statistics about the OT&E actions that have been taken by a particular
+   * registrar.
    */
   public static OteStats getFromRegistrar(String registrarName) {
     return new OteStats().recordRegistrarHistory(registrarName);

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -30,7 +30,7 @@ public class UpdateAutoTimestamp extends ImmutableObject {
 
   DateTime timestamp;
 
-  /** Returns the timestamp, or {@link START_OF_TIME} if it's null. */
+  /** Returns the timestamp, or {@code START_OF_TIME} if it's null. */
   public DateTime getTimestamp() {
     return Optional.ofNullable(timestamp).orElse(START_OF_TIME);
   }

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -30,6 +30,7 @@ import com.google.common.collect.Sets;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
 import com.googlecode.objectify.annotation.Parent;
@@ -43,14 +44,28 @@ import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithLongVKey;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embedded;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.Transient;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
 /** A billable event in a domain's lifecycle. */
+@MappedSuperclass
+@WithLongVKey
 public abstract class BillingEvent extends ImmutableObject
     implements Buildable, TransferServerApproveEntity {
 
@@ -93,24 +108,41 @@ public abstract class BillingEvent extends ImmutableObject
 
   /** Entity id. */
   @Id
-  long id;
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  Long id;
 
-  @Parent
-  @DoNotHydrate
-  Key<HistoryEntry> parent;
+  @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
 
   /** The registrar to bill. */
   @Index
+  @Column(nullable = false)
   String clientId;
 
+  /** Revision id of the entry in DomainHistory table that ths bill belongs to. */
+  // TODO(shicong): Add foreign key constraint when DomainHistory table is generated
+  @Ignore
+  @Column(nullable = false)
+  Long domainHistoryRevisionId;
+
+  /** ID of the EPP resource that the bill is for. */
+  // TODO(shicong): Add foreign key constraint when we expand DatastoreHelp for Postgresql
+  @Ignore
+  @Column(nullable = false)
+  String domainRepoId;
+
   /** When this event was created. For recurring events, this is also the recurrence start time. */
   @Index
+  @Column(nullable = false)
   DateTime eventTime;
 
   /** The reason for the bill. */
+  @Enumerated(EnumType.STRING)
+  @Column(nullable = false)
   Reason reason;
 
   /** The fully qualified domain name of the domain that the bill is for. */
+  @Column(name = "domain_name", nullable = false)
   String targetId;
 
   @Nullable
@@ -120,6 +152,14 @@ public abstract class BillingEvent extends ImmutableObject
     return clientId;
   }
 
+  public long getDomainHistoryRevisionId() {
+    return domainHistoryRevisionId;
+  }
+
+  public String getDomainRepoId() {
+    return domainRepoId;
+  }
+
   public DateTime getEventTime() {
     return eventTime;
   }
@@ -163,7 +203,7 @@ public abstract class BillingEvent extends ImmutableObject
       return thisCastToDerived();
     }
 
-    public B setId(Long id) {
+    public B setId(long id) {
       getInstance().id = id;
       return thisCastToDerived();
     }
@@ -173,6 +213,16 @@ public abstract class BillingEvent extends ImmutableObject
       return thisCastToDerived();
     }
 
+    public B setDomainHistoryRevisionId(long domainHistoryRevisionId) {
+      getInstance().domainHistoryRevisionId = domainHistoryRevisionId;
+      return thisCastToDerived();
+    }
+
+    public B setDomainRepoId(String domainRepoId) {
+      getInstance().domainRepoId = domainRepoId;
+      return thisCastToDerived();
+    }
+
     public B setEventTime(DateTime eventTime) {
       getInstance().eventTime = eventTime;
       return thisCastToDerived();
@@ -194,6 +244,7 @@ public abstract class BillingEvent extends ImmutableObject
     }
 
     public B setParent(Key<HistoryEntry> parentKey) {
+      // TODO(shicong): Figure out how to set domainHistoryRevisionId and domainRepoId
       getInstance().parent = parentKey;
       return thisCastToDerived();
     }
@@ -213,9 +264,23 @@ public abstract class BillingEvent extends ImmutableObject
   /** A one-time billable event. */
   @ReportedOn
   @Entity
+  @javax.persistence.Entity(name = "BillingEvent")
+  @javax.persistence.Table(
+      indexes = {
+        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "eventTime"),
+        @javax.persistence.Index(columnList = "billingTime"),
+        @javax.persistence.Index(columnList = "syntheticCreationTime"),
+        @javax.persistence.Index(columnList = "allocation_token_id")
+      })
+  @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
   public static class OneTime extends BillingEvent {
 
     /** The billable value. */
+    @AttributeOverrides({
+      @AttributeOverride(name = "money.amount", column = @Column(name = "cost_amount")),
+      @AttributeOverride(name = "money.currency", column = @Column(name = "cost_currency"))
+    })
     Money cost;
 
     /** When the cost should be billed. */
@@ -223,8 +288,8 @@ public abstract class BillingEvent extends ImmutableObject
     DateTime billingTime;
 
     /**
-     * The period in years of the action being billed for, if applicable, otherwise null.
-     * Used for financial reporting.
+     * The period in years of the action being billed for, if applicable, otherwise null. Used for
+     * financial reporting.
      */
     @IgnoreSave(IfNull.class)
     Integer periodYears = null;
@@ -240,15 +305,21 @@ public abstract class BillingEvent extends ImmutableObject
 
     /**
      * For {@link Flag#SYNTHETIC} events, a {@link Key} to the {@link BillingEvent} from which this
-     * OneTime was created. This is needed in order to properly match billing events against
-     * {@link Cancellation}s.
+     * OneTime was created. This is needed in order to properly match billing events against {@link
+     * Cancellation}s.
      */
-    Key<? extends BillingEvent> cancellationMatchingBillingEvent;
+    @Column(name = "cancellation_matching_billing_recurrence_id")
+    VKey<? extends BillingEvent> cancellationMatchingBillingEvent;
 
     /**
      * The {@link AllocationToken} used in the creation of this event, or null if one was not used.
+     *
+     * <p>TODO(shicong): Add foreign key constraint when AllocationToken schema is generated
      */
-    @Index @Nullable Key<AllocationToken> allocationToken;
+    @Column(name = "allocation_token_id")
+    @Index
+    @Nullable
+    VKey<AllocationToken> allocationToken;
 
     public Money getCost() {
       return cost;
@@ -266,14 +337,18 @@ public abstract class BillingEvent extends ImmutableObject
       return syntheticCreationTime;
     }
 
-    public Key<? extends BillingEvent> getCancellationMatchingBillingEvent() {
+    public VKey<? extends BillingEvent> getCancellationMatchingBillingEvent() {
       return cancellationMatchingBillingEvent;
     }
 
-    public Optional<Key<AllocationToken>> getAllocationToken() {
+    public Optional<VKey<AllocationToken>> getAllocationToken() {
       return Optional.ofNullable(allocationToken);
     }
 
+    public VKey<OneTime> createVKey() {
+      return VKey.createOfy(getClass(), Key.create(this));
+    }
+
     @Override
     public Builder asBuilder() {
       return new Builder(clone(this));
@@ -311,12 +386,12 @@ public abstract class BillingEvent extends ImmutableObject
       }
 
       public Builder setCancellationMatchingBillingEvent(
-          Key<? extends BillingEvent> cancellationMatchingBillingEvent) {
+          VKey<? extends BillingEvent> cancellationMatchingBillingEvent) {
         getInstance().cancellationMatchingBillingEvent = cancellationMatchingBillingEvent;
         return this;
       }
 
-      public Builder setAllocationToken(@Nullable Key<AllocationToken> allocationToken) {
+      public Builder setAllocationToken(@Nullable VKey<AllocationToken> allocationToken) {
         getInstance().allocationToken = allocationToken;
         return this;
       }
@@ -361,6 +436,15 @@ public abstract class BillingEvent extends ImmutableObject
    */
   @ReportedOn
   @Entity
+  @javax.persistence.Entity(name = "BillingRecurrence")
+  @javax.persistence.Table(
+      indexes = {
+        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "eventTime"),
+        @javax.persistence.Index(columnList = "recurrenceEndTime"),
+        @javax.persistence.Index(columnList = "recurrence_time_of_year")
+      })
+  @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
   public static class Recurring extends BillingEvent {
 
     /**
@@ -384,6 +468,10 @@ public abstract class BillingEvent extends ImmutableObject
      * model, whereas the billing time is a fixed {@link org.joda.time.Duration} later.
      */
     @Index
+    @Embedded
+    @AttributeOverrides({
+      @AttributeOverride(name = "timeString", column = @Column(name = "recurrence_time_of_year"))
+    })
     TimeOfYear recurrenceTimeOfYear;
 
     public DateTime getRecurrenceEndTime() {
@@ -394,6 +482,10 @@ public abstract class BillingEvent extends ImmutableObject
       return recurrenceTimeOfYear;
     }
 
+    public VKey<Recurring> createVKey() {
+      return VKey.createOfy(getClass(), Key.create(this));
+    }
+
     @Override
     public Builder asBuilder() {
       return new Builder(clone(this));
@@ -434,6 +526,14 @@ public abstract class BillingEvent extends ImmutableObject
    */
   @ReportedOn
   @Entity
+  @javax.persistence.Entity(name = "BillingCancellation")
+  @javax.persistence.Table(
+      indexes = {
+        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "eventTime"),
+        @javax.persistence.Index(columnList = "billingTime")
+      })
+  @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
   public static class Cancellation extends BillingEvent {
 
     /** The billing time of the charge that is being cancelled. */
@@ -446,7 +546,8 @@ public abstract class BillingEvent extends ImmutableObject
      * <p>Although the type is {@link Key} the name "ref" is preserved for historical reasons.
      */
     @IgnoreSave(IfNull.class)
-    Key<BillingEvent.OneTime> refOneTime = null;
+    @Column(name = "billing_event_id")
+    VKey<BillingEvent.OneTime> refOneTime = null;
 
     /**
      * The recurring billing event to cancel, or null for non-autorenew cancellations.
@@ -454,13 +555,14 @@ public abstract class BillingEvent extends ImmutableObject
      * <p>Although the type is {@link Key} the name "ref" is preserved for historical reasons.
      */
     @IgnoreSave(IfNull.class)
-    Key<BillingEvent.Recurring> refRecurring = null;
+    @Column(name = "billing_recurrence_id")
+    VKey<BillingEvent.Recurring> refRecurring = null;
 
     public DateTime getBillingTime() {
       return billingTime;
     }
 
-    public Key<? extends BillingEvent> getEventKey() {
+    public VKey<? extends BillingEvent> getEventKey() {
       return firstNonNull(refOneTime, refRecurring);
     }
 
@@ -492,13 +594,19 @@ public abstract class BillingEvent extends ImmutableObject
           .setParent(historyEntry);
       // Set the grace period's billing event using the appropriate Cancellation builder method.
       if (gracePeriod.getOneTimeBillingEvent() != null) {
-        builder.setOneTimeEventKey(gracePeriod.getOneTimeBillingEvent());
+        builder.setOneTimeEventKey(
+            VKey.createOfy(BillingEvent.OneTime.class, gracePeriod.getOneTimeBillingEvent()));
       } else if (gracePeriod.getRecurringBillingEvent() != null) {
-        builder.setRecurringEventKey(gracePeriod.getRecurringBillingEvent());
+        builder.setRecurringEventKey(
+            VKey.createOfy(BillingEvent.Recurring.class, gracePeriod.getRecurringBillingEvent()));
       }
       return builder.build();
     }
 
+    public VKey<Cancellation> createVKey() {
+      return VKey.createOfy(getClass(), Key.create(this));
+    }
+
     @Override
     public Builder asBuilder() {
       return new Builder(clone(this));
@@ -518,12 +626,12 @@ public abstract class BillingEvent extends ImmutableObject
         return this;
       }
 
-      public Builder setOneTimeEventKey(Key<BillingEvent.OneTime> eventKey) {
+      public Builder setOneTimeEventKey(VKey<BillingEvent.OneTime> eventKey) {
         getInstance().refOneTime = eventKey;
         return this;
       }
 
-      public Builder setRecurringEventKey(Key<BillingEvent.Recurring> eventKey) {
+      public Builder setRecurringEventKey(VKey<BillingEvent.Recurring> eventKey) {
         getInstance().refRecurring = eventKey;
         return this;
       }
@@ -540,9 +648,7 @@ public abstract class BillingEvent extends ImmutableObject
     }
   }
 
-  /**
-   * An event representing a modification of an existing one-time billing event.
-   */
+  /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
   public static class Modification extends BillingEvent {

core/src/main/java/google/registry/model/common/Cursor.java

@@ -21,6 +21,7 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -28,6 +29,8 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.registry.Registry;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.List;
 import org.joda.time.DateTime;
 
@@ -37,7 +40,7 @@ import org.joda.time.DateTime;
  * as scoped on {@link EntityGroupRoot}.
  */
 @Entity
-public class Cursor extends ImmutableObject {
+public class Cursor extends ImmutableObject implements DatastoreEntity {
 
   /** The types of cursors, used as the string id field for each cursor in Datastore. */
   public enum CursorType {
@@ -134,6 +137,11 @@ public class Cursor extends ImmutableObject {
     return CursorType.valueOf(String.join("_", id.subList(1, id.size())));
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // Cursors are not converted since they are ephemeral
+  }
+
   /**
    * Checks that the type of the scoped object (or null) matches the required type for the specified
    * cursor (or null, if the cursor is a global cursor).

core/src/main/java/google/registry/model/common/EntityGroupRoot.java

@@ -14,10 +14,13 @@
 
 package google.registry.model.common;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.BackupGroupRoot;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /**
  * The root key for the entity group which is known as the cross-tld entity group for historical
@@ -34,7 +37,7 @@ import google.registry.model.BackupGroupRoot;
  * the entity group for the single namespace where global data applicable for all TLDs lived.
  */
 @Entity
-public class EntityGroupRoot extends BackupGroupRoot {
+public class EntityGroupRoot extends BackupGroupRoot implements DatastoreEntity {
 
   @SuppressWarnings("unused")
   @Id
@@ -44,4 +47,9 @@ public class EntityGroupRoot extends BackupGroupRoot {
   public static Key<EntityGroupRoot> getCrossTldKey() {
     return Key.create(EntityGroupRoot.class, "cross-tld");
   }
+
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
 }

core/src/main/java/google/registry/model/common/TimeOfYear.java

@@ -29,20 +29,22 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
 import java.util.List;
+import javax.persistence.Embeddable;
 import org.joda.time.DateTime;
 
 /**
  * A time of year (month, day, millis of day) that can be stored in a sort-friendly format.
  *
- * <p>This is conceptually similar to {@code MonthDay} in Joda or more generally to Joda's
- * {@code Partial}, but the parts we need are too simple to justify a full implementation of
- * {@code Partial}.
+ * <p>This is conceptually similar to {@code MonthDay} in Joda or more generally to Joda's {@code
+ * Partial}, but the parts we need are too simple to justify a full implementation of {@code
+ * Partial}.
  *
  * <p>For simplicity, the native representation of this class's data is its stored format. This
  * allows it to be embeddable with no translation needed and also delays parsing of the string on
  * load until it's actually needed.
  */
 @Embed
+@Embeddable
 public class TimeOfYear extends ImmutableObject {
 
   /**

core/src/main/java/google/registry/model/contact/ContactAddress.java

@@ -16,13 +16,14 @@ package google.registry.model.contact;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.eppcommon.Address;
+import javax.persistence.Embeddable;
 
 /**
  * EPP Contact Address
  *
- * <p>This class is embedded inside the {@link PostalInfo} of an EPP contact to hold its
- * address. The fields are all defined in parent class {@link Address}, but the subclass is still
- * necessary to pick up the contact namespace.
+ * <p>This class is embedded inside the {@link PostalInfo} of an EPP contact to hold its address.
+ * The fields are all defined in parent class {@link Address}, but the subclass is still necessary
+ * to pick up the contact namespace.
  *
  * <p>This does not implement {@code Overlayable} because it is intended to be bulk replaced on
  * update.
@@ -30,6 +31,7 @@ import google.registry.model.eppcommon.Address;
  * @see PostalInfo
  */
 @Embed
+@Embeddable
 public class ContactAddress extends Address {
 
   /** Builder for {@link ContactAddress}. */

core/src/main/java/google/registry/model/contact/ContactAuthInfo.java

@@ -20,6 +20,7 @@ import javax.xml.bind.annotation.XmlType;
 
 /** A version of authInfo specifically for contacts. */
 @Embed
+@javax.persistence.Embeddable
 @XmlType(namespace = "urn:ietf:params:xml:ns:contact-1.0")
 public class ContactAuthInfo extends AuthInfo {
   public static ContactAuthInfo create(PasswordAuth pw) {

core/src/main/java/google/registry/model/contact/ContactCommand.java

@@ -37,7 +37,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 /** A collection of {@link ContactResource} commands. */
 public class ContactCommand {
 
-  /** The fields on "chgType" from {@link "http://tools.ietf.org/html/rfc5733"}. */
+  /** The fields on "chgType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
   @XmlTransient
   public static class ContactCreateOrChange extends ImmutableObject
       implements ResourceCreateOrChange<ContactResource.Builder> {
@@ -111,8 +111,8 @@ public class ContactCommand {
   }
 
   /**
-   * A create command for a {@link ContactResource}, mapping "createType" from
-   * {@link "http://tools.ietf.org/html/rfc5733"}.
+   * A create command for a {@link ContactResource}, mapping "createType" from <a
+   * href="http://tools.ietf.org/html/rfc5733">RFC5733</a>}.
    */
   @XmlType(propOrder = {"contactId", "postalInfo", "voice", "fax", "email", "authInfo", "disclose"})
   @XmlRootElement

core/src/main/java/google/registry/model/contact/ContactPhoneNumber.java

@@ -16,17 +16,19 @@ package google.registry.model.contact;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.eppcommon.PhoneNumber;
+import javax.persistence.Embeddable;
 
 /**
  * EPP Contact Phone Number
  *
  * <p>This class is embedded inside a {@link ContactResource} hold the phone number of an EPP
- * contact.  The fields are all defined in the parent class {@link PhoneNumber}, but the subclass is
+ * contact. The fields are all defined in the parent class {@link PhoneNumber}, but the subclass is
  * still necessary to pick up the contact namespace.
  *
  * @see ContactResource
  */
 @Embed
+@Embeddable
 public class ContactPhoneNumber extends PhoneNumber {
 
   /** Builder for {@link ContactPhoneNumber}. */

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -19,6 +19,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
 
 import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
@@ -30,9 +31,17 @@ import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.PostalInfo.Type;
 import google.registry.model.transfer.TransferData;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Stream;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embedded;
+import javax.persistence.Transient;
 import javax.xml.bind.annotation.XmlElement;
 import org.joda.time.DateTime;
 
@@ -43,9 +52,20 @@ import org.joda.time.DateTime;
  */
 @ReportedOn
 @Entity
+@javax.persistence.Entity(name = "Contact")
+@javax.persistence.Table(
+    name = "Contact",
+    indexes = {
+      @javax.persistence.Index(columnList = "creationTime"),
+      @javax.persistence.Index(columnList = "currentSponsorClientId"),
+      @javax.persistence.Index(columnList = "deletionTime"),
+      @javax.persistence.Index(columnList = "contactId", unique = true),
+      @javax.persistence.Index(columnList = "searchName")
+    })
 @ExternalMessagingName("contact")
-public class ContactResource extends EppResource implements
-    ForeignKeyedEppResource, ResourceWithTransferData {
+@WithStringVKey
+public class ContactResource extends EppResource
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource, ResourceWithTransferData {
 
   /**
    * Unique identifier for this contact.
@@ -61,13 +81,55 @@ public class ContactResource extends EppResource implements
    * US-ASCII character set. Personal info; cleared by {@link Builder#wipeOut}.
    */
   @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_local_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_local_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_local_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_local_country_code"))
+  })
   PostalInfo localizedPostalInfo;
 
   /**
-   * Internationalized postal info for the contact. Personal info; cleared by
-   * {@link Builder#wipeOut}.
+   * Internationalized postal info for the contact. Personal info; cleared by {@link
+   * Builder#wipeOut}.
    */
   @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_i18n_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_i18n_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_i18n_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_i18n_country_code"))
+  })
   PostalInfo internationalizedPostalInfo;
 
   /**
@@ -80,10 +142,20 @@ public class ContactResource extends EppResource implements
 
   /** Contact’s voice number. Personal info; cleared by {@link Builder#wipeOut}. */
   @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
+  })
   ContactPhoneNumber voice;
 
   /** Contact’s fax number. Personal info; cleared by {@link Builder#wipeOut}. */
   @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
+  })
   ContactPhoneNumber fax;
 
   /** Contact’s email address. Personal info; cleared by {@link Builder#wipeOut}. */
@@ -91,10 +163,16 @@ public class ContactResource extends EppResource implements
   String email;
 
   /** Authorization info (aka transfer secret) of the contact. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
+    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
+  })
   ContactAuthInfo authInfo;
 
   /** Data about any pending or past transfers on this contact. */
-  TransferData transferData;
+  // TODO(b/153363295): Figure out how to persist transfer data
+  @Transient TransferData transferData;
 
   /**
    * The time that this resource was last transferred.
@@ -107,8 +185,24 @@ public class ContactResource extends EppResource implements
   // the wipeOut() function, so that data is not kept around for deleted contacts.
 
   /** Disclosure policy. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
+    @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
+    @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
+    @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
+    @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
+    @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
+  })
   Disclose disclose;
 
+  @Override
+  public VKey<ContactResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.createOfy(ContactResource.class, Key.create(this));
+  }
+
   public String getContactId() {
     return contactId;
   }

core/src/main/java/google/registry/model/contact/Disclose.java

@@ -22,11 +22,14 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.eppcommon.PresenceMarker;
 import java.util.List;
+import javax.persistence.Embeddable;
+import javax.persistence.Embedded;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlType;
 
-/** The "discloseType" from {@link "http://tools.ietf.org/html/rfc5733"}. */
+/** The "discloseType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
 @Embed
+@Embeddable
 @XmlType(propOrder = {"name", "org", "addr", "voice", "fax", "email"})
 public class Disclose extends ImmutableObject {
 
@@ -36,11 +39,11 @@ public class Disclose extends ImmutableObject {
 
   List<PostalInfoChoice> addr;
 
-  PresenceMarker voice;
+  @Embedded PresenceMarker voice;
 
-  PresenceMarker fax;
+  @Embedded PresenceMarker fax;
 
-  PresenceMarker email;
+  @Embedded PresenceMarker email;
 
   @XmlAttribute
   Boolean flag;
@@ -73,7 +76,7 @@ public class Disclose extends ImmutableObject {
     return flag;
   }
 
-  /** The "intLocType" from {@link "http://tools.ietf.org/html/rfc5733"}. */
+  /** The "intLocType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
   @Embed
   public static class PostalInfoChoice extends ImmutableObject {
     @XmlAttribute

core/src/main/java/google/registry/model/contact/PostalInfo.java

@@ -21,6 +21,9 @@ import google.registry.model.Buildable;
 import google.registry.model.Buildable.Overlayable;
 import google.registry.model.ImmutableObject;
 import java.util.Optional;
+import javax.persistence.Embeddable;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlEnumValue;
@@ -29,10 +32,11 @@ import javax.xml.bind.annotation.adapters.NormalizedStringAdapter;
 import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 /**
- * Implementation of both "postalInfoType" and "chgPostalInfoType" from
- * {@link "http://tools.ietf.org/html/rfc5733"}.
+ * Implementation of both "postalInfoType" and "chgPostalInfoType" from <a href=
+ * "http://tools.ietf.org/html/rfc5733">RFC5733</a>.
  */
 @Embed
+@Embeddable
 @XmlType(propOrder = {"name", "org", "address", "type"})
 public class PostalInfo extends ImmutableObject implements Overlayable<PostalInfo> {
 
@@ -53,6 +57,7 @@ public class PostalInfo extends ImmutableObject implements Overlayable<PostalInf
   @XmlElement(name = "addr")
   ContactAddress address;
 
+  @Enumerated(EnumType.STRING)
   @XmlAttribute
   Type type;
 

core/src/main/java/google/registry/model/domain/DesignatedContact.java

@@ -18,9 +18,11 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
 import google.registry.model.contact.ContactResource;
+import google.registry.persistence.VKey;
 import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlEnumValue;
 
@@ -36,6 +38,9 @@ import javax.xml.bind.annotation.XmlEnumValue;
  * situation with hosts where client-side renames would make that data stale. However, we sometimes
  * rename contacts internally ourselves, and it's easier to use the same model for both cases.
  *
+ * <p>This entity type is not persisted in Cloud SQL. The different roles are represented as
+ * separate fields in the Domain table.
+ *
  * @see <a href="http://tools.ietf.org/html/rfc5731#section-2.2">RFC 5731 - EPP Domain Name Mapping
  *     - Contact and Client Identifiers</a>
  */
@@ -58,22 +63,28 @@ public class DesignatedContact extends ImmutableObject {
     REGISTRANT
   }
 
-  public static DesignatedContact create(Type type, Key<ContactResource> contact) {
+  public static DesignatedContact create(Type type, VKey<ContactResource> contact) {
     DesignatedContact instance = new DesignatedContact();
     instance.type = type;
-    instance.contact = checkArgumentNotNull(contact, "Must specify contact key");
+    instance.contactVKey = checkArgumentNotNull(contact, "Must specify contact key");
+    instance.contact = contact.maybeGetOfyKey().orElse(null);
     return instance;
   }
 
   Type type;
 
   @Index Key<ContactResource> contact;
+  @Ignore VKey<ContactResource> contactVKey;
 
   public Type getType() {
     return type;
   }
 
-  public Key<ContactResource> getContactKey() {
-    return contact;
+  public VKey<ContactResource> getContactKey() {
+    return contactVKey;
+  }
+
+  public DesignatedContact reconstitute() {
+    return create(type, VKey.createOfy(ContactResource.class, contact));
   }
 }

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -66,6 +66,7 @@ import google.registry.model.registry.Registry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.util.CollectionUtils;
 import java.util.HashSet;
 import java.util.Objects;
@@ -78,6 +79,8 @@ import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.ElementCollection;
 import javax.persistence.Embedded;
+import javax.persistence.JoinTable;
+import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
@@ -105,7 +108,7 @@ import org.joda.time.Interval;
     })
 @ExternalMessagingName("domain")
 public class DomainBase extends EppResource
-    implements ForeignKeyedEppResource, ResourceWithTransferData {
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource, ResourceWithTransferData {
 
   /** The max number of years that a domain can be registered for, as set by ICANN policy. */
   public static final int MAX_REGISTRATION_YEARS = 10;
@@ -133,15 +136,11 @@ public class DomainBase extends EppResource
   @Index
   String tld;
 
-  /**
-   * References to hosts that are the nameservers for the domain.
-   *
-   * <p>This is a legacy field: we have to preserve it because it is still persisted and indexed in
-   * the datastore, but all external references go through nsHostVKeys.
-   */
-  @Index @ElementCollection @Transient Set<Key<HostResource>> nsHosts;
-
-  @Ignore @Transient Set<VKey<HostResource>> nsHostVKeys;
+  /** References to hosts that are the nameservers for the domain. */
+  @Index
+  @ElementCollection
+  @JoinTable(name = "DomainHost")
+  Set<VKey<HostResource>> nsHosts;
 
   /**
    * The union of the contacts visible via {@link #getContacts} and {@link #getRegistrant}.
@@ -150,6 +149,17 @@ public class DomainBase extends EppResource
    */
   @Transient Set<DesignatedContact> allContacts;
 
+  /**
+   * Contacts as they are stored in cloud SQL.
+   *
+   * <p>This information is duplicated in allContacts, and must be kept in sync with it.
+   */
+  @Ignore VKey<ContactResource> adminContact;
+
+  @Ignore VKey<ContactResource> billingContact;
+  @Ignore VKey<ContactResource> techContact;
+  @Ignore VKey<ContactResource> registrantContact;
+
   /** Authorization info (aka transfer secret) of the domain. */
   @Embedded
   @AttributeOverrides({
@@ -252,10 +262,33 @@ public class DomainBase extends EppResource
 
   @OnLoad
   void load() {
-    nsHostVKeys =
-        nullToEmptyImmutableCopy(nsHosts).stream()
-            .map(hostKey -> VKey.createOfy(HostResource.class, hostKey))
-            .collect(toImmutableSet());
+    // Reconstitute all of the contacts so that they have VKeys.
+    allContacts =
+        allContacts.stream().map(contact -> contact.reconstitute()).collect(toImmutableSet());
+    setContactFields(allContacts, true);
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Reconstitute the contact list.
+    ImmutableSet.Builder<DesignatedContact> contactsBuilder =
+        new ImmutableSet.Builder<DesignatedContact>();
+
+    if (registrantContact != null) {
+      contactsBuilder.add(
+          DesignatedContact.create(DesignatedContact.Type.REGISTRANT, registrantContact));
+    }
+    if (billingContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.BILLING, billingContact));
+    }
+    if (techContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.TECH, techContact));
+    }
+    if (adminContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.ADMIN, adminContact));
+    }
+
+    allContacts = contactsBuilder.build();
   }
 
   public ImmutableSet<String> getSubordinateHosts() {
@@ -318,9 +351,7 @@ public class DomainBase extends EppResource
   }
 
   public ImmutableSet<VKey<HostResource>> getNameservers() {
-    // Since nsHostVKeys gets initialized both from setNameservers() and the OnLoad method, this
-    // should always be valid.
-    return nullToEmptyImmutableCopy(nsHostVKeys);
+    return nullToEmptyImmutableCopy(nsHosts);
   }
 
   public final String getCurrentSponsorClientId() {
@@ -510,13 +541,20 @@ public class DomainBase extends EppResource
   }
 
   /** A key to the registrant who registered this domain. */
-  public Key<ContactResource> getRegistrant() {
-    return nullToEmpty(allContacts)
-        .stream()
-        .filter(IS_REGISTRANT)
-        .findFirst()
-        .get()
-        .getContactKey();
+  public VKey<ContactResource> getRegistrant() {
+    return registrantContact;
+  }
+
+  public VKey<ContactResource> getAdminContact() {
+    return adminContact;
+  }
+
+  public VKey<ContactResource> getBillingContact() {
+    return billingContact;
+  }
+
+  public VKey<ContactResource> getTechContact() {
+    return techContact;
   }
 
   /** Associated contacts for the domain (other than registrant). */
@@ -532,7 +570,7 @@ public class DomainBase extends EppResource
   }
 
   /** Returns all referenced contacts from this domain or application. */
-  public ImmutableSet<Key<ContactResource>> getReferencedContacts() {
+  public ImmutableSet<VKey<ContactResource>> getReferencedContacts() {
     return nullToEmptyImmutableCopy(allContacts)
         .stream()
         .map(DesignatedContact::getContactKey)
@@ -544,6 +582,43 @@ public class DomainBase extends EppResource
     return tld;
   }
 
+  /**
+   * Sets the individual contact fields from {@code contacts}.
+   *
+   * <p>The registrant field is only set if {@code includeRegistrant} is true, as this field needs
+   * to be set in some circumstances but not in others.
+   */
+  private void setContactFields(Set<DesignatedContact> contacts, boolean includeRegistrant) {
+
+    // Set the individual contact fields.
+    for (DesignatedContact contact : contacts) {
+      switch (contact.getType()) {
+        case BILLING:
+          billingContact = contact.getContactKey();
+          break;
+        case TECH:
+          techContact = contact.getContactKey();
+          break;
+        case ADMIN:
+          adminContact = contact.getContactKey();
+          break;
+        case REGISTRANT:
+          if (includeRegistrant) {
+            registrantContact = contact.getContactKey();
+          }
+          break;
+        default:
+          throw new IllegalArgumentException("Unknown contact resource type: " + contact.getType());
+      }
+    }
+  }
+
+  @Override
+  public VKey<DomainBase> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.createOfy(DomainBase.class, Key.create(this));
+  }
+
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
   private static final Predicate<DesignatedContact> IS_REGISTRANT =
       (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);
@@ -562,14 +637,6 @@ public class DomainBase extends EppResource
 
     Builder(DomainBase instance) {
       super(instance);
-
-      // Convert nsHosts to nsHostVKeys.
-      if (instance.nsHosts != null) {
-        instance.nsHostVKeys =
-            instance.nsHosts.stream()
-                .map(key -> VKey.createOfy(HostResource.class, key))
-                .collect(toImmutableSet());
-      }
     }
 
     @Override
@@ -588,7 +655,11 @@ public class DomainBase extends EppResource
 
       checkArgumentNotNull(
           emptyToNull(instance.fullyQualifiedDomainName), "Missing fullyQualifiedDomainName");
-      checkArgument(instance.allContacts.stream().anyMatch(IS_REGISTRANT), "Missing registrant");
+      if (instance.getRegistrant() == null
+          && instance.allContacts.stream().anyMatch(IS_REGISTRANT)) {
+        throw new IllegalArgumentException("registrant is null but is in allContacts");
+      }
+      checkArgumentNotNull(instance.getRegistrant(), "Missing registrant");
       instance.tld = getTldFromDomainName(instance.fullyQualifiedDomainName);
       return super.build();
     }
@@ -606,11 +677,14 @@ public class DomainBase extends EppResource
       return thisCastToDerived();
     }
 
-    public Builder setRegistrant(Key<ContactResource> registrant) {
+    public Builder setRegistrant(VKey<ContactResource> registrant) {
       // Replace the registrant contact inside allContacts.
       getInstance().allContacts = union(
           getInstance().getContacts(),
           DesignatedContact.create(Type.REGISTRANT, checkArgumentNotNull(registrant)));
+
+      // Set the registrant field specifically.
+      getInstance().registrantContact = registrant;
       return thisCastToDerived();
     }
 
@@ -620,27 +694,12 @@ public class DomainBase extends EppResource
     }
 
     public Builder setNameservers(VKey<HostResource> nameserver) {
-      Optional<Key<HostResource>> nsKey = nameserver.maybeGetOfyKey();
-      if (nsKey.isPresent()) {
-        getInstance().nsHosts = ImmutableSet.of(nsKey.get());
-      } else {
-        getInstance().nsHosts = null;
-      }
-      getInstance().nsHostVKeys = ImmutableSet.of(nameserver);
+      getInstance().nsHosts = ImmutableSet.of(nameserver);
       return thisCastToDerived();
     }
 
     public Builder setNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
-      // If we have all of the ofy keys, we can set nsHosts.  Otherwise, make it null.
-      if (nameservers != null
-          && nameservers.stream().allMatch(key -> key.maybeGetOfyKey().isPresent())) {
-        getInstance().nsHosts =
-            nameservers.stream().map(key -> key.getOfyKey()).collect(toImmutableSet());
-      } else {
-        getInstance().nsHosts = null;
-      }
-
-      getInstance().nsHostVKeys = forceEmptyToNull(nameservers);
+      getInstance().nsHosts = forceEmptyToNull(nameservers);
       return thisCastToDerived();
     }
 
@@ -668,12 +727,16 @@ public class DomainBase extends EppResource
 
     public Builder setContacts(ImmutableSet<DesignatedContact> contacts) {
       checkArgument(contacts.stream().noneMatch(IS_REGISTRANT), "Registrant cannot be a contact");
+
       // Replace the non-registrant contacts inside allContacts.
       getInstance().allContacts =
           Streams.concat(
                   nullToEmpty(getInstance().allContacts).stream().filter(IS_REGISTRANT),
                   contacts.stream())
               .collect(toImmutableSet());
+
+      // Set the individual fields.
+      getInstance().setContactFields(contacts, false);
       return thisCastToDerived();
     }
 

core/src/main/java/google/registry/model/domain/DomainCommand.java

@@ -30,7 +30,6 @@ import com.google.common.base.MoreObjects;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.contact.ContactResource;
@@ -41,6 +40,7 @@ import google.registry.model.eppinput.ResourceCommand.ResourceUpdate;
 import google.registry.model.eppinput.ResourceCommand.SingleResourceCommand;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
+import google.registry.persistence.VKey;
 import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
@@ -71,18 +71,17 @@ public class DomainCommand {
     T cloneAndLinkReferences(DateTime now) throws InvalidReferencesException;
   }
 
-  /** The fields on "chgType" from {@link "http://tools.ietf.org/html/rfc5731"}. */
+  /** The fields on "chgType" from <a href="http://tools.ietf.org/html/rfc5731">RFC5731</a>. */
   @XmlTransient
-  public static class DomainCreateOrChange<B extends DomainBase.Builder>
-      extends ImmutableObject implements ResourceCreateOrChange<B> {
+  public static class DomainCreateOrChange<B extends DomainBase.Builder> extends ImmutableObject
+      implements ResourceCreateOrChange<B> {
 
     /** The contactId of the registrant who registered this domain. */
     @XmlElement(name = "registrant")
     String registrantContactId;
 
     /** A resolved key to the registrant who registered this domain. */
-    @XmlTransient
-    Key<ContactResource> registrant;
+    @XmlTransient VKey<ContactResource> registrant;
 
     /** Authorization info (aka transfer secret) of the domain. */
     DomainAuthInfo authInfo;
@@ -92,7 +91,7 @@ public class DomainCommand {
     }
 
     @Nullable
-    public Key<ContactResource> getRegistrant() {
+    public VKey<ContactResource> getRegistrant() {
       return registrant;
     }
 
@@ -102,19 +101,20 @@ public class DomainCommand {
   }
 
   /**
-   * A create command for a {@link DomainBase}, mapping "createType" from
-   * {@link "http://tools.ietf.org/html/rfc5731"}.
+   * A create command for a {@link DomainBase}, mapping "createType" from <a
+   * href="http://tools.ietf.org/html/rfc5731">RFC5731</a>.
    */
   @XmlRootElement
-  @XmlType(propOrder = {
-      "fullyQualifiedDomainName",
-      "period",
-      "nameserverFullyQualifiedHostNames",
-      "registrantContactId",
-      "foreignKeyedDesignatedContacts",
-      "authInfo"})
-  public static class Create
-      extends DomainCreateOrChange<DomainBase.Builder>
+  @XmlType(
+      propOrder = {
+        "fullyQualifiedDomainName",
+        "period",
+        "nameserverFullyQualifiedHostNames",
+        "registrantContactId",
+        "foreignKeyedDesignatedContacts",
+        "authInfo"
+      })
+  public static class Create extends DomainCreateOrChange<DomainBase.Builder>
       implements CreateOrUpdate<Create> {
 
     /** Fully qualified domain name, which serves as a unique identifier for this domain. */
@@ -127,8 +127,7 @@ public class DomainCommand {
     Set<String> nameserverFullyQualifiedHostNames;
 
     /** Resolved keys to hosts that are the nameservers for the domain. */
-    @XmlTransient
-    Set<Key<HostResource>> nameservers;
+    @XmlTransient Set<VKey<HostResource>> nameservers;
 
     /** Foreign keyed associated contacts for the domain (other than registrant). */
     @XmlElement(name = "contact")
@@ -158,7 +157,7 @@ public class DomainCommand {
       return nullToEmptyImmutableCopy(nameserverFullyQualifiedHostNames);
     }
 
-    public ImmutableSet<Key<HostResource>> getNameservers() {
+    public ImmutableSet<VKey<HostResource>> getNameservers() {
       return nullToEmptyImmutableCopy(nameservers);
     }
 
@@ -352,8 +351,7 @@ public class DomainCommand {
       Set<String> nameserverFullyQualifiedHostNames;
 
       /** Resolved keys to hosts that are the nameservers for the domain. */
-      @XmlTransient
-      Set<Key<HostResource>> nameservers;
+      @XmlTransient Set<VKey<HostResource>> nameservers;
 
       /** Foreign keyed associated contacts for the domain (other than registrant). */
       @XmlElement(name = "contact")
@@ -367,7 +365,7 @@ public class DomainCommand {
         return nullSafeImmutableCopy(nameserverFullyQualifiedHostNames);
       }
 
-      public ImmutableSet<Key<HostResource>> getNameservers() {
+      public ImmutableSet<VKey<HostResource>> getNameservers() {
         return nullToEmptyImmutableCopy(nameservers);
       }
 
@@ -417,7 +415,7 @@ public class DomainCommand {
     }
   }
 
-  private static Set<Key<HostResource>> linkHosts(
+  private static Set<VKey<HostResource>> linkHosts(
       Set<String> fullyQualifiedHostNames, DateTime now) throws InvalidReferencesException {
     if (fullyQualifiedHostNames == null) {
       return null;
@@ -435,18 +433,18 @@ public class DomainCommand {
     for (ForeignKeyedDesignatedContact contact : contacts) {
       foreignKeys.add(contact.contactId);
     }
-    ImmutableMap<String, Key<ContactResource>> loadedContacts =
+    ImmutableMap<String, VKey<ContactResource>> loadedContacts =
         loadByForeignKeysCached(foreignKeys.build(), ContactResource.class, now);
     ImmutableSet.Builder<DesignatedContact> linkedContacts = new ImmutableSet.Builder<>();
     for (ForeignKeyedDesignatedContact contact : contacts) {
-      linkedContacts.add(DesignatedContact.create(
-          contact.type, loadedContacts.get(contact.contactId)));
+      linkedContacts.add(
+          DesignatedContact.create(contact.type, loadedContacts.get(contact.contactId)));
     }
     return linkedContacts.build();
   }
 
   /** Loads keys to cached EPP resources by their foreign keys. */
-  private static <T extends EppResource> ImmutableMap<String, Key<T>> loadByForeignKeysCached(
+  private static <T extends EppResource> ImmutableMap<String, VKey<T>> loadByForeignKeysCached(
       final Set<String> foreignKeys, final Class<T> clazz, final DateTime now)
       throws InvalidReferencesException {
     Map<String, ForeignKeyIndex<T>> fkis = ForeignKeyIndex.loadCached(clazz, foreignKeys, now);

core/src/main/java/google/registry/model/domain/Period.java

@@ -20,7 +20,7 @@ import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlEnumValue;
 import javax.xml.bind.annotation.XmlValue;
 
-/** The "periodType" from {@link "http://tools.ietf.org/html/rfc5731"}. */
+/** The "periodType" from <a href="http://tools.ietf.org/html/rfc5731">RFC5731</a>. */
 @Embed
 @javax.persistence.Embeddable
 public class Period extends ImmutableObject {

core/src/main/java/google/registry/model/domain/fee12/FeeCheckCommandExtensionItemV12.java

@@ -29,11 +29,13 @@ import org.joda.time.DateTime;
  * An individual price check item in version 0.12 of the fee extension on domain check commands.
  * Items look like:
  *
+ * <pre>{@code
  * <fee:command name="renew" phase="sunrise" subphase="hello">
  *   <fee:period unit="y">1</fee:period>
  *   <fee:class>premium</fee:class>
  *   <fee:date>2017-05-17T13:22:21.0Z</fee:date>
  * </fee:command>
+ * }</pre>
  *
  * In a change from previous versions of the extension, items do not contain domain names; instead,
  * the names from the non-extension check element are used.

core/src/main/java/google/registry/model/domain/launch/LaunchNotice.java

@@ -95,9 +95,6 @@ public class LaunchNotice extends ImmutableObject {
 
   /**
    * Validate the checksum of the notice against the domain label.
-   *
-   * @throws IllegalArgumentException
-   * @throws InvalidChecksumException
    */
   public void validate(String domainLabel) throws InvalidChecksumException {
     // According to http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3, a TCNID

core/src/main/java/google/registry/model/domain/rgp/GracePeriodStatus.java

@@ -49,9 +49,9 @@ public enum GracePeriodStatus implements EppEnum {
   AUTO_RENEW("autoRenewPeriod"),
 
   /**
-   * This status value is used to describe a domain for which a <delete> command has been received,
-   * but the domain has not yet been purged because an opportunity exists to restore the domain and
-   * abort the deletion process.
+   * This status value is used to describe a domain for which a &lt;delete&gt; command has been
+   * received, but the domain has not yet been purged because an opportunity exists to restore the
+   * domain and abort the deletion process.
    */
   REDEMPTION("redemptionPeriod"),
 

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -45,6 +45,8 @@ import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.common.TimedTransitionProperty.TimeMapper;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
@@ -53,6 +55,7 @@ import org.joda.time.DateTime;
 /** An entity representing an allocation token. */
 @ReportedOn
 @Entity
+@WithStringVKey
 public class AllocationToken extends BackupGroupRoot implements Buildable {
 
   // Promotions should only move forward, and ENDED / CANCELLED are terminal states.
@@ -179,6 +182,10 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
     return tokenStatusTransitions;
   }
 
+  public VKey<AllocationToken> createVKey() {
+    return VKey.createOfy(getClass(), Key.create(this));
+  }
+
   @Override
   public Builder asBuilder() {
     return new Builder(clone(this));

core/src/main/java/google/registry/model/domain/token/AllocationTokenExtension.java

@@ -25,7 +25,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
  * An allocation token extension that may be present on EPP domain commands.
  *
  * @see <a href="https://tools.ietf.org/html/draft-ietf-regext-allocation-token-04">the IETF
- *     draft</a> for full details.
+ *     draft</a>
  */
 @XmlRootElement(name = "allocationToken")
 public class AllocationTokenExtension extends ImmutableObject implements CommandExtension {

core/src/main/java/google/registry/model/eppcommon/Address.java

@@ -44,8 +44,9 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 /**
  * Container for generic street address.
  *
- * <p>This is the "addrType" type from {@link "http://tools.ietf.org/html/rfc5733"}. It also matches
- * the "addrType" type from {@link "http://tools.ietf.org/html/draft-lozano-tmch-smd"}.
+ * <p>This is the "addrType" type from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. It
+ * also matches the "addrType" type from <a
+ * href="http://tools.ietf.org/html/draft-lozano-tmch-smd">Mark and Signed Mark Objects Mapping</a>.
  *
  * @see google.registry.model.contact.ContactAddress
  * @see google.registry.model.mark.MarkAddress

core/src/main/java/google/registry/model/eppcommon/PhoneNumber.java

@@ -18,6 +18,8 @@ import static com.google.common.base.Preconditions.checkNotNull;
 
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import javax.persistence.Embeddable;
+import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlTransient;
 import javax.xml.bind.annotation.XmlValue;
@@ -27,21 +29,26 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 /**
  * Container for generic E164 phone number.
  *
- * <p>This is the "e164" type from {@link "http://tools.ietf.org/html/rfc5733"}. It also matches the
- * "e164Type" type from {@link "http://tools.ietf.org/html/draft-lozano-tmch-smd"}.
+ * <p>This is the "e164" type from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. It also
+ * matches the "e164Type" type from <a href="http://tools.ietf.org/html/draft-lozano-tmch-smd">Mark
+ * and Signed Mark Objects Mapping</a>
  *
  * <blockquote>
+ *
  * <p>"Contact telephone number structure is derived from structures defined in [ITU.E164.2005].
  * Telephone numbers described in this mapping are character strings that MUST begin with a plus
  * sign ("+", ASCII value 0x002B), followed by a country code defined in [ITU.E164.2005], followed
  * by a dot (".", ASCII value 0x002E), followed by a sequence of digits representing the telephone
  * number. An optional "x" attribute is provided to note telephone extension information."
+ *
  * </blockquote>
  *
  * @see google.registry.model.contact.ContactPhoneNumber
  * @see google.registry.model.mark.MarkPhoneNumber
  */
 @XmlTransient
+@Embeddable
+@MappedSuperclass
 public class PhoneNumber extends ImmutableObject {
 
   @XmlValue

core/src/main/java/google/registry/model/eppcommon/PresenceMarker.java

@@ -17,6 +17,7 @@ package google.registry.model.eppcommon;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
 import java.io.Serializable;
+import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlTransient;
 
 /**
@@ -26,6 +27,7 @@ import javax.xml.bind.annotation.XmlTransient;
  * {@code <foo></foo>}, and will unmarshal always to {@code <foo/>}.
  */
 @Embed
+@Embeddable
 public class PresenceMarker extends ImmutableObject implements Serializable {
   @XmlTransient
   boolean marked = true;

core/src/main/java/google/registry/model/eppoutput/Greeting.java

@@ -24,7 +24,7 @@ import javax.xml.bind.annotation.XmlElementWrapper;
 import org.joda.time.DateTime;
 
 /**
- * A greeting, defined in {@link "http://tools.ietf.org/html/rfc5730"}.
+ * A greeting, defined in <a href="http://tools.ietf.org/html/rfc5730">RFC5730</a>.
  *
  * <p>It would be nice to make this a singleton, but we need the {@link #svDate} field to stay
  * current.

core/src/main/java/google/registry/model/host/HostCommand.java

@@ -32,7 +32,7 @@ import javax.xml.bind.annotation.XmlType;
 /** A collection of {@link HostResource} commands. */
 public class HostCommand {
 
-  /** The fields on "chgType" from {@link "http://tools.ietf.org/html/rfc5732"}. */
+  /** The fields on "chgType" from <a href="http://tools.ietf.org/html/rfc5732">RFC5732</a>. */
   @XmlTransient
   abstract static class HostCreateOrChange extends AbstractSingleResourceCommand
       implements ResourceCreateOrChange<HostResource.Builder> {
@@ -42,13 +42,13 @@ public class HostCommand {
   }
 
   /**
-   * A create command for a {@link HostResource}, mapping "createType" from
-   * {@link "http://tools.ietf.org/html/rfc5732"}.
+   * A create command for a {@link HostResource}, mapping "createType" from <a
+   * href="http://tools.ietf.org/html/rfc5732">RFC5732</a>.
    */
-  @XmlType(propOrder = {"targetId", "inetAddresses" })
+  @XmlType(propOrder = {"targetId", "inetAddresses"})
   @XmlRootElement
-  public static class Create
-      extends HostCreateOrChange implements ResourceCreateOrChange<HostResource.Builder> {
+  public static class Create extends HostCreateOrChange
+      implements ResourceCreateOrChange<HostResource.Builder> {
     /** IP Addresses for this host. Can be null if this is an external host. */
     @XmlElement(name = "addr")
     Set<InetAddress> inetAddresses;

core/src/main/java/google/registry/model/host/HostResource.java

@@ -34,24 +34,30 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.transfer.TransferData;
 import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.net.InetAddress;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
+import javax.persistence.ElementCollection;
 import org.joda.time.DateTime;
 
 /**
  * A persistable Host resource including mutable and non-mutable fields.
  *
- * <p>A host's {@link TransferData} is stored on the superordinate domain.  Non-subordinate hosts
+ * <p>A host's {@link TransferData} is stored on the superordinate domain. Non-subordinate hosts
  * don't carry a full set of TransferData; all they have is lastTransferTime.
  *
  * @see <a href="https://tools.ietf.org/html/rfc5732">RFC 5732</a>
  */
 @ReportedOn
 @Entity
+@javax.persistence.Entity
 @ExternalMessagingName("host")
-public class HostResource extends EppResource implements ForeignKeyedEppResource {
+@WithStringVKey
+public class HostResource extends EppResource
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource {
 
   /**
    * Fully qualified hostname, which is a unique identifier for this host.
@@ -64,8 +70,7 @@ public class HostResource extends EppResource implements ForeignKeyedEppResource
   String fullyQualifiedHostName;
 
   /** IP Addresses for this host. Can be null if this is an external host. */
-  @Index
-  Set<InetAddress> inetAddresses;
+  @Index @ElementCollection Set<InetAddress> inetAddresses;
 
   /** The superordinate domain of this host, or null if this is an external host. */
   @Index
@@ -118,7 +123,9 @@ public class HostResource extends EppResource implements ForeignKeyedEppResource
     return fullyQualifiedHostName;
   }
 
-  public VKey<HostResource> createKey() {
+  @Override
+  public VKey<HostResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.createOfy(HostResource.class, Key.create(this));
   }
 

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -43,6 +43,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
 import java.util.Optional;
@@ -99,7 +100,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * <p>This field holds a key to the only referenced resource. It is named "topReference" for
    * historical reasons.
    */
-  Key<E> topReference;
+  VKey<E> topReference;
 
   public String getForeignKey() {
     return foreignKey;
@@ -109,7 +110,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     return deletionTime;
   }
 
-  public Key<E> getResourceKey() {
+  public VKey<E> getResourceKey() {
     return topReference;
   }
 
@@ -125,7 +126,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     @SuppressWarnings("unchecked")
     Class<E> resourceClass = (Class<E>) resource.getClass();
     ForeignKeyIndex<E> instance = instantiate(mapToFkiClass(resourceClass));
-    instance.topReference = Key.create(resource);
+    instance.topReference = (VKey<E>) resource.createVKey();
     instance.foreignKey = resource.getForeignKey();
     instance.deletionTime = deletionTime;
     return instance;
@@ -141,18 +142,18 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   /**
    * Loads a {@link Key} to an {@link EppResource} from Datastore by foreign key.
    *
-   * <p>Returns null if no foreign key index with this foreign key was ever created, or if the
-   * most recently created foreign key index was deleted before time "now". This method does not
-   * actually check that the referenced resource actually exists. However, for normal epp resources,
-   * it is safe to assume that the referenced resource exists if the foreign key index does.
+   * <p>Returns null if no foreign key index with this foreign key was ever created, or if the most
+   * recently created foreign key index was deleted before time "now". This method does not actually
+   * check that the referenced resource actually exists. However, for normal epp resources, it is
+   * safe to assume that the referenced resource exists if the foreign key index does.
    *
    * @param clazz the resource type to load
    * @param foreignKey id to match
    * @param now the current logical time to use when checking for soft deletion of the foreign key
-   *        index
+   *     index
    */
   @Nullable
-  public static <E extends EppResource> Key<E> loadAndGetKey(
+  public static <E extends EppResource> VKey<E> loadAndGetKey(
       Class<E> clazz, String foreignKey, DateTime now) {
     ForeignKeyIndex<E> index = load(clazz, foreignKey, now);
     return (index == null) ? null : index.getResourceKey();

core/src/main/java/google/registry/model/ofy/CommitLogBucket.java

@@ -22,6 +22,7 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.collect.ContiguousSet;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Range;
@@ -33,6 +34,8 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.util.NonFinalForTesting;
 import java.util.Random;
 import java.util.function.Supplier;
@@ -51,7 +54,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogBucket extends ImmutableObject implements Buildable {
+public class CommitLogBucket extends ImmutableObject implements Buildable, DatastoreEntity {
 
   /**
    * Ranges from 1 to {@link RegistryConfig#getCommitLogBucketCount()}, inclusive; starts at 1 since
@@ -70,6 +73,11 @@ public class CommitLogBucket extends ImmutableObject implements Buildable {
     return lastWrittenTime;
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   /**
    * Returns the key for the specified bucket ID.
    *

core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java

@@ -27,6 +27,8 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -44,7 +46,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogCheckpoint extends ImmutableObject {
+public class CommitLogCheckpoint extends ImmutableObject implements DatastoreEntity {
 
   /** Shared singleton parent entity for commit log checkpoints. */
   @Parent
@@ -71,6 +73,11 @@ public class CommitLogCheckpoint extends ImmutableObject {
     return builder.build();
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   /**
    * Creates a CommitLogCheckpoint for the given wall time and bucket checkpoint times, specified as
    * a map from bucket ID to bucket commit timestamp.

core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java

@@ -17,12 +17,15 @@ package google.registry.model.ofy;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import org.joda.time.DateTime;
 
 /**
@@ -30,7 +33,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogCheckpointRoot extends ImmutableObject {
+public class CommitLogCheckpointRoot extends ImmutableObject implements DatastoreEntity {
 
   public static final long SINGLETON_ID = 1;  // There is always exactly one of these.
 
@@ -49,6 +52,11 @@ public class CommitLogCheckpointRoot extends ImmutableObject {
     return lastWrittenTime;
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   public static CommitLogCheckpointRoot loadRoot() {
     CommitLogCheckpointRoot root = ofy().load().key(getKey()).now();
     return root == null ? new CommitLogCheckpointRoot() : root;

core/src/main/java/google/registry/model/ofy/CommitLogManifest.java

@@ -17,6 +17,7 @@ package google.registry.model.ofy;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static org.joda.time.DateTimeZone.UTC;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -25,6 +26,8 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.LinkedHashSet;
 import java.util.Set;
 import org.joda.time.DateTime;
@@ -38,7 +41,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogManifest extends ImmutableObject {
+public class CommitLogManifest extends ImmutableObject implements DatastoreEntity {
 
   /** Commit log manifests are parented on a random bucket. */
   @Parent
@@ -67,6 +70,11 @@ public class CommitLogManifest extends ImmutableObject {
     return nullToEmptyImmutableCopy(deletions);
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   public static CommitLogManifest create(
       Key<CommitLogBucket> parent, DateTime commitTime, Set<Key<?>> deletions) {
     CommitLogManifest instance = new CommitLogManifest();

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -21,6 +21,7 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 
 import com.google.appengine.api.datastore.KeyFactory;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -28,11 +29,13 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /** Representation of a saved entity in a {@link CommitLogManifest} (not deletes). */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogMutation extends ImmutableObject {
+public class CommitLogMutation extends ImmutableObject implements DatastoreEntity {
 
   /** The manifest this belongs to. */
   @Parent
@@ -58,6 +61,11 @@ public class CommitLogMutation extends ImmutableObject {
     return createFromPbBytes(entityProtoBytes);
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   /**
    * Returns a new mutation entity created from an @Entity ImmutableObject instance.
    *

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -14,7 +14,9 @@
 
 package google.registry.model.ofy;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
@@ -22,6 +24,7 @@ import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.TransactionManager;
 import java.util.Iterator;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.StreamSupport;
@@ -93,42 +96,45 @@ public class DatastoreTransactionManager implements TransactionManager {
 
   @Override
   public void saveNew(Object entity) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    checkArgumentNotNull(entity, "entity must be specified");
+    getOfy().save().entity(entity);
   }
 
   @Override
   public void saveAllNew(ImmutableCollection<?> entities) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    getOfy().save().entities(entities);
   }
 
   @Override
   public void saveNewOrUpdate(Object entity) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    checkArgumentNotNull(entity, "entity must be specified");
+    getOfy().save().entity(entity);
   }
 
   @Override
   public void saveNewOrUpdateAll(ImmutableCollection<?> entities) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    getOfy().save().entities(entities);
   }
 
   @Override
   public void update(Object entity) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    checkArgumentNotNull(entity, "entity must be specified");
+    getOfy().save().entity(entity);
   }
 
   @Override
   public void updateAll(ImmutableCollection<?> entities) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    getOfy().save().entities(entities);
   }
 
   @Override
   public boolean checkExists(Object entity) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    return getOfy().load().key(Key.create(entity)).now() != null;
   }
 
   @Override
   public <T> boolean checkExists(VKey<T> key) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    return getOfy().load().key(key.getOfyKey()).now() != null;
   }
 
   // TODO: add tests for these methods.  They currently have some degree of test coverage because
@@ -136,14 +142,23 @@ public class DatastoreTransactionManager implements TransactionManager {
   // VKey instead of by ofy Key.  But ideally, there should be one set of TransactionManager
   // interface tests that are applied to both the datastore and SQL implementations.
   @Override
-  public <T> Optional<T> load(VKey<T> key) {
-    return Optional.of(getOfy().load().key(key.getOfyKey()).now());
+  public <T> Optional<T> maybeLoad(VKey<T> key) {
+    return Optional.ofNullable(getOfy().load().key(key.getOfyKey()).now());
+  }
+
+  @Override
+  public <T> T load(VKey<T> key) {
+    T result = getOfy().load().key(key.getOfyKey()).now();
+    if (result == null) {
+      throw new NoSuchElementException(key.toString());
+    }
+    return result;
   }
 
   @Override
   public <T> ImmutableList<T> load(Iterable<VKey<T>> keys) {
     Iterator<Key<T>> iter =
-        StreamSupport.stream(keys.spliterator(), false).map(key -> key.getOfyKey()).iterator();
+        StreamSupport.stream(keys.spliterator(), false).map(VKey::getOfyKey).iterator();
 
     // The lambda argument to keys() effectively converts Iterator -> Iterable.
     return ImmutableList.copyOf(getOfy().load().keys(() -> iter).values());
@@ -151,16 +166,23 @@ public class DatastoreTransactionManager implements TransactionManager {
 
   @Override
   public <T> ImmutableList<T> loadAll(Class<T> clazz) {
+    // We can do a ofy().load().type(clazz), but this doesn't work in a transaction.
     throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
   }
 
   @Override
-  public <T> int delete(VKey<T> key) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+  public void delete(VKey<?> key) {
+    getOfy().delete().key(key.getOfyKey()).now();
   }
 
   @Override
-  public <T> void assertDelete(VKey<T> key) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+  public void delete(Iterable<? extends VKey<?>> vKeys) {
+    // We have to create a list to work around the wildcard capture issue here.
+    // See https://docs.oracle.com/javase/tutorial/java/generics/capture.html
+    ImmutableList<Key<?>> list =
+        StreamSupport.stream(vKeys.spliterator(), false)
+            .map(VKey::getOfyKey)
+            .collect(toImmutableList());
+    getOfy().delete().keys(list).now();
   }
 }

core/src/main/java/google/registry/model/ofy/ObjectifyService.java

@@ -34,9 +34,9 @@ import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.impl.translate.TranslatorFactory;
 import com.googlecode.objectify.impl.translate.opt.joda.MoneyStringTranslatorFactory;
 import google.registry.config.RegistryEnvironment;
+import google.registry.model.Buildable;
 import google.registry.model.EntityClasses;
 import google.registry.model.ImmutableObject;
-import google.registry.model.host.HostResource;
 import google.registry.model.translators.BloomFilterOfStringTranslatorFactory;
 import google.registry.model.translators.CidrAddressBlockTranslatorFactory;
 import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
@@ -130,7 +130,7 @@ public class ObjectifyService {
             new InetAddressTranslatorFactory(),
             new MoneyStringTranslatorFactory(),
             new ReadableInstantUtcTranslatorFactory(),
-            new VKeyTranslatorFactory<HostResource>(HostResource.class),
+            new VKeyTranslatorFactory(),
             new UpdateAutoTimestampTranslatorFactory())) {
       factory().getTranslators().add(translatorFactory);
     }
@@ -151,11 +151,14 @@ public class ObjectifyService {
       String kind = Key.getKind(clazz);
       boolean registered = factory().getMetadata(kind) != null;
       if (clazz.isAnnotationPresent(Entity.class)) {
-        // Objectify silently ignores re-registrations for a given kind string, even if the classes
-        // being registered are distinct. Throw an exception if that would happen here.
-        checkState(!registered,
+        // Objectify silently replaces current registration for a given kind string when a different
+        // class is registered again for this kind. For simplicity's sake, throw an exception on any
+        // re-registration.
+        checkState(
+            !registered,
             "Kind '%s' already registered, cannot register new @Entity %s",
-            kind, clazz.getCanonicalName());
+            kind,
+            clazz.getCanonicalName());
       } else if (clazz.isAnnotationPresent(EntitySubclass.class)) {
         // Ensure that any @EntitySubclass classes have also had their parent @Entity registered,
         // which Objectify nominally requires but doesn't enforce in 4.x (though it may in 5.x).
@@ -165,10 +168,16 @@ public class ObjectifyService {
       }
       com.googlecode.objectify.ObjectifyService.register(clazz);
       // Autogenerated ids make the commit log code very difficult since we won't always be able
-      // to create a key for an entity immediately when requesting a save. Disallow that here.
-      checkState(
-          !factory().getMetadata(clazz).getKeyMetadata().isIdGeneratable(),
-          "Can't register %s: Autogenerated ids (@Id on a Long) are not supported.", kind);
+      // to create a key for an entity immediately when requesting a save. So, we require such
+      // entities to implement google.registry.model.Buildable as its build() function allocates the
+      // id to the entity.
+      if (factory().getMetadata(clazz).getKeyMetadata().isIdGeneratable()) {
+        checkState(
+            Buildable.class.isAssignableFrom(clazz),
+            "Can't register %s: Entity with autogenerated ids (@Id on a Long) must implement"
+                + " google.registry.model.Buildable.",
+            kind);
+      }
     }
   }
 

core/src/main/java/google/registry/model/poll/PendingActionNotificationResponse.java

@@ -19,6 +19,7 @@ import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
+import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
@@ -29,11 +30,12 @@ import org.joda.time.DateTime;
 
 /** The {@link ResponseData} returned when completing a pending action on a domain. */
 @XmlTransient
-public abstract class PendingActionNotificationResponse
-    extends ImmutableObject implements ResponseData {
+@Embeddable
+public class PendingActionNotificationResponse extends ImmutableObject implements ResponseData {
 
   /** The inner name type that contains a name and the result boolean. */
   @Embed
+  @Embeddable
   static class NameOrId extends ImmutableObject {
     @XmlValue
     String value;

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -25,6 +25,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
@@ -39,10 +40,23 @@ import google.registry.model.poll.PendingActionNotificationResponse.DomainPendin
 import google.registry.model.poll.PendingActionNotificationResponse.HostPendingActionNotificationResponse;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
+import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferResponse.ContactTransferResponse;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
+import google.registry.persistence.WithLongVKey;
 import java.util.List;
 import java.util.Optional;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.DiscriminatorColumn;
+import javax.persistence.DiscriminatorValue;
+import javax.persistence.Embedded;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Inheritance;
+import javax.persistence.InheritanceType;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -62,34 +76,58 @@ import org.joda.time.DateTime;
  * <p>Poll messages are identified externally by registrars using the format defined in {@link
  * PollMessageExternalKeyConverter}.
  *
- * @see <a href="https://tools.ietf.org/html/rfc5730#section-2.9.2.3">RFC5730 - EPP - &ltpoll&gt
+ * @see <a href="https://tools.ietf.org/html/rfc5730#section-2.9.2.3">RFC5730 - EPP - &lt;poll&gt;
  *     Command</a>
  */
 @Entity
 @ReportedOn
 @ExternalMessagingName("message")
+@javax.persistence.Entity
+@Inheritance(strategy = InheritanceType.SINGLE_TABLE)
+@DiscriminatorColumn(name = "type")
+@javax.persistence.Table(
+    indexes = {
+      @javax.persistence.Index(columnList = "registrar_id"),
+      @javax.persistence.Index(columnList = "eventTime")
+    })
 public abstract class PollMessage extends ImmutableObject
     implements Buildable, TransferServerApproveEntity {
 
   /** Entity id. */
   @Id
-  long id;
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  @Column(name = "poll_message_id")
+  Long id;
 
-  @Parent
-  @DoNotHydrate
-  Key<HistoryEntry> parent;
+  @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
 
   /** The registrar that this poll message will be delivered to. */
   @Index
+  @Column(name = "registrar_id", nullable = false)
   String clientId;
 
   /** The time when the poll message should be delivered. May be in the future. */
   @Index
+  @Column(nullable = false)
   DateTime eventTime;
 
   /** Human readable message that will be returned with this poll message. */
+  @Column(name = "message")
   String msg;
 
+  @Ignore String domainRepoId;
+
+  @Ignore String contactRepoId;
+
+  @Ignore String hostRepoId;
+
+  @Ignore Long domainRevisionId;
+
+  @Ignore Long contactRevisionId;
+
+  @Ignore Long hostRevisionId;
+
   public Key<HistoryEntry> getParentKey() {
     return parent;
   }
@@ -180,15 +218,78 @@ public abstract class PollMessage extends ImmutableObject
    * <p>One-time poll messages are deleted from Datastore once they have been delivered and ACKed.
    */
   @EntitySubclass(index = false)
+  @javax.persistence.Entity
+  @DiscriminatorValue("ONE_TIME")
+  @WithLongVKey
   public static class OneTime extends PollMessage {
 
     // Response data. Objectify cannot persist a base class type, so we must have a separate field
     // to hold every possible derived type of ResponseData that we might store.
+    @Transient
     List<ContactPendingActionNotificationResponse> contactPendingActionNotificationResponses;
-    List<ContactTransferResponse> contactTransferResponses;
+
+    @Transient List<ContactTransferResponse> contactTransferResponses;
+
+    @Transient
     List<DomainPendingActionNotificationResponse> domainPendingActionNotificationResponses;
-    List<DomainTransferResponse> domainTransferResponses;
-    List<HostPendingActionNotificationResponse> hostPendingActionNotificationResponses;
+
+    @Transient List<DomainTransferResponse> domainTransferResponses;
+
+    @Transient List<HostPendingActionNotificationResponse> hostPendingActionNotificationResponses;
+
+    @Ignore
+    @Embedded
+    @AttributeOverrides({
+      @AttributeOverride(
+          name = "nameOrId.value",
+          column = @Column(name = "pending_action_response_name_or_id")),
+      @AttributeOverride(
+          name = "nameOrId.actionResult",
+          column = @Column(name = "pending_action_response_action_result")),
+      @AttributeOverride(
+          name = "trid.serverTransactionId",
+          column = @Column(name = "pending_action_response_server_txn_id")),
+      @AttributeOverride(
+          name = "trid.clientTransactionId",
+          column = @Column(name = "pending_action_response_client_txn_id")),
+      @AttributeOverride(
+          name = "processedDate",
+          column = @Column(name = "pending_action_response_processed_date"))
+    })
+    PendingActionNotificationResponse pendingActionNotificationResponse;
+
+    @Ignore
+    @Embedded
+    @AttributeOverrides({
+      @AttributeOverride(
+          name = "transferStatus",
+          column = @Column(name = "transfer_response_transfer_status")),
+      @AttributeOverride(
+          name = "gainingClientId",
+          column = @Column(name = "transfer_response_gaining_registrar_id")),
+      @AttributeOverride(
+          name = "transferRequestTime",
+          column = @Column(name = "transfer_response_transfer_request_time")),
+      @AttributeOverride(
+          name = "losingClientId",
+          column = @Column(name = "transfer_response_losing_registrar_id")),
+      @AttributeOverride(
+          name = "pendingTransferExpirationTime",
+          column = @Column(name = "transfer_response_pending_transfer_expiration_time"))
+    })
+    TransferResponse transferResponse;
+
+    @Ignore
+    @Column(name = "transfer_response_domain_name")
+    String fullyQualifiedDomainName;
+
+    @Ignore
+    @Column(name = "transfer_response_domain_expiration_time")
+    DateTime extendedRegistrationExpirationTime;
+
+    @Ignore
+    @Column(name = "transfer_response_contact_id")
+    String contactId;
 
     @Override
     public Builder asBuilder() {
@@ -265,9 +366,13 @@ public abstract class PollMessage extends ImmutableObject
    * happens.
    */
   @EntitySubclass(index = false)
+  @javax.persistence.Entity
+  @DiscriminatorValue("AUTORENEW")
+  @WithLongVKey
   public static class Autorenew extends PollMessage {
 
     /** The target id of the autorenew event. */
+    @Column(name = "autorenew_domain_name")
     String targetId;
 
     /** The autorenew recurs annually between {@link #eventTime} and this time. */

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -73,6 +73,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.registrar.Registrar.BillingAccountEntry.CurrencyMapper;
 import google.registry.model.registry.Registry;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.util.CidrAddressBlock;
 import java.security.cert.CertificateParsingException;
 import java.util.Comparator;
@@ -107,11 +108,12 @@ import org.joda.time.DateTime;
           columnList = "ianaIdentifier",
           name = "registrar_iana_identifier_idx"),
     })
-public class Registrar extends ImmutableObject implements Buildable, Jsonifiable {
+public class Registrar extends ImmutableObject
+    implements Buildable, DatastoreAndSqlEntity, Jsonifiable {
 
   /** Represents the type of a registrar entity. */
   public enum Type {
-    /** A real-world, third-party registrar.  Should have non-null IANA and billing IDs. */
+    /** A real-world, third-party registrar. Should have non-null IANA and billing IDs. */
     REAL(Objects::nonNull),
 
     /**
@@ -392,8 +394,6 @@ public class Registrar extends ImmutableObject implements Buildable, Jsonifiable
    */
   @Nullable
   @Mapify(CurrencyMapper.class)
-  @org.hibernate.annotations.Type(
-      type = "google.registry.persistence.converter.CurrencyToBillingMapUserType")
   Map<CurrencyUnit, BillingAccountEntry> billingAccountMap;
 
   /** A billing account entry for this registrar, consisting of a currency and an account Id. */

core/src/main/java/google/registry/model/registrar/RegistrarContact.java

@@ -42,6 +42,7 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Optional;
@@ -68,7 +69,8 @@ import javax.persistence.Transient;
       @javax.persistence.Index(columnList = "gaeUserId", name = "registrarpoc_gae_user_id_idx")
     })
 // TODO(shicong): Rename the class name to RegistrarPoc after database migration
-public class RegistrarContact extends ImmutableObject implements Jsonifiable {
+public class RegistrarContact extends ImmutableObject
+    implements DatastoreAndSqlEntity, Jsonifiable {
 
   @Parent @Transient Key<Registrar> parent;
 

core/src/main/java/google/registry/model/registry/label/DomainLabelEntry.java

@@ -19,6 +19,7 @@ import static com.google.common.base.Strings.emptyToNull;
 import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.net.InternetDomainName;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
@@ -83,6 +84,13 @@ public abstract class DomainLabelEntry<T extends Comparable<?>, D extends Domain
           "Label '%s' must be in puny-coded, lower-case form",
           getInstance().label);
       checkArgumentNotNull(getInstance().getValue(), "Value must be specified");
+      // Verify that the label creates a valid SLD if we add a TLD to the end of it.
+      // We require that the label is not already a full domain name including a dot.
+      // Domain name validation is tricky, so let InternetDomainName handle it for us.
+      checkArgument(
+          InternetDomainName.from(getInstance().label + ".tld").parts().size() == 2,
+          "Label %s must not be a multi-level domain name",
+          getInstance().label);
       return super.build();
     }
   }

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -31,6 +31,7 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
 import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
 import com.google.common.hash.BloomFilter;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.googlecode.objectify.Key;
@@ -41,6 +42,8 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registry.Registry;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.util.NonFinalForTesting;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -53,26 +56,28 @@ import javax.annotation.Nullable;
 import org.joda.money.Money;
 import org.joda.time.Duration;
 
-/**
- * A premium list entity, persisted to Datastore, that is used to check domain label prices.
- */
+/** A premium list entity, persisted to Datastore, that is used to check domain label prices. */
 @ReportedOn
 @Entity
-public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.PremiumListEntry> {
+public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.PremiumListEntry>
+    implements DatastoreEntity {
 
   /** Stores the revision key for the set of currently used premium list entry entities. */
   Key<PremiumListRevision> revisionKey;
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // PremiumList is dual-written
+  }
+
   /** Virtual parent entity for premium list entry entities associated with a single revision. */
   @ReportedOn
   @Entity
   public static class PremiumListRevision extends ImmutableObject {
 
-    @Parent
-    Key<PremiumList> parent;
+    @Parent Key<PremiumList> parent;
 
-    @Id
-    long revisionId;
+    @Id long revisionId;
 
     /**
      * A Bloom filter that is used to determine efficiently and quickly whether a label might be
@@ -249,7 +254,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   @ReportedOn
   @Entity
   public static class PremiumListEntry extends DomainLabelEntry<Money, PremiumListEntry>
-      implements Buildable {
+      implements Buildable, DatastoreEntity {
 
     @Parent
     Key<PremiumListRevision> parent;
@@ -266,6 +271,11 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
       return new Builder(clone(this));
     }
 
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return null;
+    }
+
     /** A builder for constructing {@link PremiumListEntry} objects, since they are immutable. */
     public static class Builder extends DomainLabelEntry.Builder<PremiumListEntry, Builder> {
 

core/src/main/java/google/registry/model/registry/label/ReservedList.java

@@ -30,6 +30,7 @@ import com.google.common.base.Splitter;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.MapDifference;
@@ -45,6 +46,8 @@ import com.googlecode.objectify.mapper.Mapper;
 import google.registry.model.Buildable;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.label.DomainLabelMetrics.MetricsReservedListMatch;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.schema.tld.ReservedList.ReservedEntry;
 import google.registry.schema.tld.ReservedListDao;
 import java.util.List;
@@ -59,7 +62,8 @@ import org.joda.time.DateTime;
  */
 @Entity
 public final class ReservedList
-    extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry> {
+    extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry> implements
+    DatastoreEntity {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -68,6 +72,11 @@ public final class ReservedList
 
   boolean shouldPublish = true;
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // ReservedList is dual-written
+  }
+
   /**
    * A reserved list entry entity, persisted to Datastore, that represents a single label and its
    * reservation type.
@@ -149,8 +158,8 @@ public final class ReservedList
   /**
    * Gets a ReservedList by name using the caching layer.
    *
-   * @return An Optional<ReservedList> that has a value if a reserved list exists by the given name,
-   *     or absent if not.
+   * @return An Optional&lt;ReservedList&gt; that has a value if a reserved list exists by the given
+   *     name, or absent if not.
    * @throws UncheckedExecutionException if some other error occurs while trying to load the
    *     ReservedList from the cache or Datastore.
    */

core/src/main/java/google/registry/model/reporting/IcannReportingTypes.java

@@ -22,8 +22,8 @@ public final class IcannReportingTypes {
    * Represents the set of possible ICANN Monthly Registry Functions Activity Report fields.
    *
    * <p>Refer to the <a
-   * href="https://newgtlds.icann.org/sites/default/files/agreements/agreement-approved-09jan14-en.htm#_DV_M278>ICANN
-   * registry agreement Specification 3 Section 2</a> for details.
+   * href="https://newgtlds.icann.org/sites/default/files/agreements/agreement-approved-09jan14-en.htm#_DV_M278">
+   * ICANN registry agreement Specification 3 Section 2</a> for details.
    */
   public enum ActivityReportField {
     DOMAIN_CHECK("srs-dom-check"),

core/src/main/java/google/registry/model/server/Lock.java

@@ -16,20 +16,21 @@ package google.registry.model.server;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
-import google.registry.schema.server.LockDao;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.RequestStatusCheckerImpl;
 import java.io.Serializable;
@@ -49,7 +50,7 @@ import org.joda.time.Duration;
  */
 @Entity
 @NotBackedUp(reason = Reason.TRANSIENT)
-public class Lock extends ImmutableObject implements Serializable {
+public class Lock extends ImmutableObject implements DatastoreEntity, Serializable {
 
   private static final long serialVersionUID = 756397280691684645L;
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -197,22 +198,6 @@ public class Lock extends ImmutableObject implements Serializable {
 
                   // Checking if an unexpired lock still exists - if so, the lock can't be acquired.
                   Lock lock = ofy().load().type(Lock.class).id(lockId).now();
-                  try {
-                    jpaTm()
-                        .transact(
-                            () -> {
-                              Optional<google.registry.schema.server.Lock> cloudSqlLockOptional;
-                              if (tld == null) {
-                                cloudSqlLockOptional = LockDao.load(resourceName);
-                              } else {
-                                cloudSqlLockOptional = LockDao.load(resourceName, tld);
-                              }
-                              LockDao.compare(Optional.ofNullable(lock), cloudSqlLockOptional);
-                            });
-                  } catch (Exception e) {
-                    logger.atSevere().withCause(e).log(
-                        "Issue loading and comparing lock from Cloud SQL");
-                  }
                   if (lock != null) {
                     logger.atInfo().log(
                         "Loaded existing lock: %s for request: %s", lock.lockId, lock.requestLogId);
@@ -237,35 +222,6 @@ public class Lock extends ImmutableObject implements Serializable {
                   // don't need to be backed up.
                   ofy().saveWithoutBackup().entity(newLock);
 
-                  // create and save the lock to Cloud SQL
-                  try {
-                    jpaTm()
-                        .transact(
-                            () -> {
-                              google.registry.schema.server.Lock cloudSqlLock;
-                              if (tld == null) {
-                                cloudSqlLock =
-                                    google.registry.schema.server.Lock.createGlobal(
-                                        resourceName,
-                                        requestStatusChecker.getLogId(),
-                                        now,
-                                        leaseLength);
-                              } else {
-                                cloudSqlLock =
-                                    google.registry.schema.server.Lock.create(
-                                        resourceName,
-                                        tld,
-                                        requestStatusChecker.getLogId(),
-                                        now,
-                                        leaseLength);
-                              }
-                              LockDao.save(cloudSqlLock);
-                            });
-                  } catch (Exception e) {
-                    logger.atSevere().withCause(e).log(
-                        "Error saving lock to Cloud SQL: %s", newLock);
-                  }
-
                   return AcquireResult.create(now, lock, newLock, lockState);
                 });
 
@@ -284,44 +240,12 @@ public class Lock extends ImmutableObject implements Serializable {
               // this can happen if release() is called around the expiration time and the lock
               // expires underneath us.
               Lock loadedLock = ofy().load().type(Lock.class).id(lockId).now();
-              try {
-                jpaTm()
-                    .transact(
-                        () -> {
-                          Optional<google.registry.schema.server.Lock> cloudSqlLockOptional;
-                          if (tld == null) {
-                            cloudSqlLockOptional = LockDao.load(resourceName);
-                          } else {
-                            cloudSqlLockOptional = LockDao.load(resourceName, tld);
-                          }
-                          LockDao.compare(Optional.ofNullable(loadedLock), cloudSqlLockOptional);
-                        });
-              } catch (Exception e) {
-                logger.atSevere().withCause(e).log(
-                    "Issue loading and comparing lock from Cloud SQL");
-              }
               if (Lock.this.equals(loadedLock)) {
                 // Use noBackupOfy() so that we don't create a commit log entry for deleting the
                 // lock.
                 logger.atInfo().log("Deleting lock: %s", lockId);
                 ofy().deleteWithoutBackup().entity(Lock.this);
 
-                // Remove the lock from Cloud SQL
-                try {
-                  jpaTm()
-                      .transact(
-                          () -> {
-                            if (tld == null) {
-                              LockDao.delete(resourceName);
-                            } else {
-                              LockDao.delete(resourceName, tld);
-                            }
-                          });
-                } catch (Exception e) {
-                  logger.atSevere().withCause(e).log(
-                      "Error deleting lock from Cloud SQL: %s", loadedLock);
-                }
-
                 lockMetrics.recordRelease(
                     resourceName, tld, new Duration(acquiredTime, tm().getTransactionTime()));
               } else {
@@ -335,4 +259,9 @@ public class Lock extends ImmutableObject implements Serializable {
               }
             });
   }
+
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // Locks are not converted since they are ephemeral
+  }
 }

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -26,6 +26,7 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.MapDifference.ValueDifference;
@@ -44,6 +45,8 @@ import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.annotations.VirtualEntity;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.schema.tmch.ClaimsList;
 import google.registry.schema.tmch.ClaimsListDao;
 import google.registry.util.CollectionUtils;
@@ -75,7 +78,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-public class ClaimsListShard extends ImmutableObject {
+public class ClaimsListShard extends ImmutableObject implements DatastoreEntity {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -289,10 +292,15 @@ public class ClaimsListShard extends ImmutableObject {
     }
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // ClaimsLists are dually written
+  }
+
   /** Virtual parent entity for claims list shards of a specific revision. */
   @Entity
   @VirtualEntity
-  public static class ClaimsListRevision extends ImmutableObject {
+  public static class ClaimsListRevision extends ImmutableObject implements DatastoreEntity {
     @Parent
     Key<ClaimsListSingleton> parent;
 
@@ -311,6 +319,11 @@ public class ClaimsListShard extends ImmutableObject {
     public static Key<ClaimsListRevision> createKey() {
       return createKey(new ClaimsListSingleton());
     }
+
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // ClaimsLists are dually written
+    }
   }
 
   /**
@@ -319,7 +332,7 @@ public class ClaimsListShard extends ImmutableObject {
    */
   @Entity
   @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-  public static class ClaimsListSingleton extends CrossTldSingleton {
+  public static class ClaimsListSingleton extends CrossTldSingleton implements DatastoreEntity {
     Key<ClaimsListRevision> activeRevision;
 
     static ClaimsListSingleton create(Key<ClaimsListRevision> revision) {
@@ -332,6 +345,11 @@ public class ClaimsListShard extends ImmutableObject {
     public void setActiveRevision(Key<ClaimsListRevision> revision) {
       activeRevision = revision;
     }
+
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // ClaimsLists are dually written
+    }
   }
 
   /**

core/src/main/java/google/registry/model/transfer/BaseTransferObject.java

@@ -16,6 +16,8 @@ package google.registry.model.transfer;
 
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlTransient;
@@ -31,6 +33,7 @@ public abstract class BaseTransferObject extends ImmutableObject {
    * will always be non-null.
    */
   @XmlElement(name = "trStatus")
+  @Enumerated(EnumType.STRING)
   TransferStatus transferStatus;
 
   /** The gaining registrar of the current or last transfer. Can be null if never transferred. */

core/src/main/java/google/registry/model/transfer/TransferResponse.java

@@ -17,6 +17,7 @@ package google.registry.model.transfer;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.EppResource;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
+import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
@@ -28,7 +29,8 @@ import org.joda.time.DateTime;
  * are common to all transfer responses; derived classes add resource specific fields.
  */
 @XmlTransient
-public abstract class TransferResponse extends BaseTransferObject implements ResponseData {
+@Embeddable
+public class TransferResponse extends BaseTransferObject implements ResponseData {
 
   /** An adapter to output the XML in response to a transfer command on a domain. */
   @Embed

core/src/main/java/google/registry/model/translators/VKeyTranslatorFactory.java

@@ -14,13 +14,14 @@
 
 package google.registry.model.translators;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
+import static com.google.common.base.Functions.identity;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static google.registry.model.EntityClasses.ALL_CLASSES;
 
-import com.googlecode.objectify.Key;
+import com.google.appengine.api.datastore.Key;
+import com.google.common.collect.ImmutableMap;
+import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.persistence.VKey;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.net.URLEncoder;
 
 /**
  * Translator factory for VKey.
@@ -28,57 +29,39 @@ import java.net.URLEncoder;
  * <p>These get translated to a string containing the URL safe encoding of the objectify key
  * followed by a (url-unsafe) ampersand delimiter and the SQL key.
  */
-public class VKeyTranslatorFactory<T> extends AbstractSimpleTranslatorFactory<VKey, String> {
-  private final Class<T> refClass;
+public class VKeyTranslatorFactory extends AbstractSimpleTranslatorFactory<VKey, Key> {
 
-  public VKeyTranslatorFactory(Class<T> refClass) {
+  // Class registry allowing us to restore the original class object from the unqualified class
+  // name, which is all the datastore key gives us.
+  // Note that entities annotated with @EntitySubclass are removed because they share the same
+  // kind of the key with their parent class.
+  private static final ImmutableMap<String, Class> CLASS_REGISTRY =
+      ALL_CLASSES.stream()
+          .filter(clazz -> !clazz.isAnnotationPresent(EntitySubclass.class))
+          .collect(toImmutableMap(com.googlecode.objectify.Key::getKind, identity()));
+  ;
+
+  public VKeyTranslatorFactory() {
     super(VKey.class);
-    this.refClass = refClass;
   }
 
   @Override
-  public SimpleTranslator<VKey, String> createTranslator() {
-    return new SimpleTranslator<VKey, String>() {
+  public SimpleTranslator<VKey, Key> createTranslator() {
+    return new SimpleTranslator<VKey, Key>() {
       @Override
-      public VKey loadValue(String datastoreValue) {
-        int pos = datastoreValue.indexOf('&');
-        Key ofyKey = null;
-        String sqlKey = null;
-        if (pos > 0) {
-          // We have an objectify key.
-          ofyKey = Key.create(datastoreValue.substring(0, pos));
-        }
-
-        if (pos < datastoreValue.length() - 1) {
-          // We have an SQL key.
-          sqlKey = decode(datastoreValue.substring(pos + 1));
-        }
-
-        return VKey.create(refClass, sqlKey, ofyKey);
+      public VKey loadValue(Key datastoreValue) {
+        // TODO(mmuller): we need to call a method on refClass to also reconstitute the SQL key.
+        return datastoreValue == null
+            ? null
+            : VKey.createOfy(
+                CLASS_REGISTRY.get(datastoreValue.getKind()),
+                com.googlecode.objectify.Key.create(datastoreValue));
       }
 
       @Override
-      public String saveValue(VKey key) {
-        return ((key.getOfyKey() == null) ? "" : key.getOfyKey().getString())
-            + "&"
-            + ((key.getSqlKey() == null) ? "" : encode(key.getSqlKey().toString()));
+      public Key saveValue(VKey key) {
+        return key == null ? null : key.getOfyKey().getRaw();
       }
     };
   }
-
-  private static String encode(String val) {
-    try {
-      return URLEncoder.encode(val, UTF_8.toString());
-    } catch (UnsupportedEncodingException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  private static String decode(String encoded) {
-    try {
-      return URLDecoder.decode(encoded, UTF_8.toString());
-    } catch (UnsupportedEncodingException e) {
-      throw new RuntimeException(e);
-    }
-  }
 }

core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java

@@ -16,6 +16,7 @@ package google.registry.module.frontend;
 
 import dagger.Module;
 import dagger.Subcomponent;
+import google.registry.batch.BatchModule;
 import google.registry.dns.DnsModule;
 import google.registry.flows.EppTlsAction;
 import google.registry.flows.FlowComponent;
@@ -38,6 +39,7 @@ import google.registry.ui.server.registrar.RegistryLockVerifyAction;
 @RequestScope
 @Subcomponent(
     modules = {
+      BatchModule.class,
       DnsModule.class,
       EppTlsModule.class,
       RegistrarConsoleModule.class,

core/src/main/java/google/registry/persistence/EntityCallbacksListener.java

@@ -0,0 +1,200 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.Stream;
+import javax.persistence.Embeddable;
+import javax.persistence.Embedded;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.PostLoad;
+import javax.persistence.PostPersist;
+import javax.persistence.PostRemove;
+import javax.persistence.PostUpdate;
+import javax.persistence.PrePersist;
+import javax.persistence.PreRemove;
+import javax.persistence.PreUpdate;
+import javax.persistence.Transient;
+
+/**
+ * A listener class to invoke entity callbacks in cases where Hibernate doesn't invoke the callback
+ * as expected.
+ *
+ * <p>JPA defines a few annotations, e.g. {@link PostLoad}, that we can use for the application to
+ * react to certain events that occur inside the persistence mechanism. However, Hibernate only
+ * supports a few basic use cases, e.g. defining a {@link PostLoad} method directly in an {@link
+ * javax.persistence.Entity} class or in an {@link Embeddable} class. If the annotated method is
+ * defined in an {@link Embeddable} class that is a property of another {@link Embeddable} class, or
+ * it is defined in a parent class of the {@link Embeddable} class, Hibernate doesn't invoke it.
+ *
+ * <p>This listener is added in core/src/main/resources/META-INF/orm.xml as a default entity
+ * listener whose annotated methods will be invoked by Hibernate when corresponding events happen.
+ * For example, {@link EntityCallbacksListener#prePersist} will be invoked before the entity is
+ * persisted to the database, then it will recursively invoke any other {@link PrePersist} method
+ * that should be invoked but not handled by Hibernate due to the bug.
+ *
+ * @see <a
+ *     href="https://docs.jboss.org/hibernate/orm/current/userguide/html_single/Hibernate_User_Guide.html#events-jpa-callbacks">JPA
+ *     Callbacks</a>
+ * @see <a href="https://hibernate.atlassian.net/browse/HHH-13316">HHH-13316</a>
+ */
+public class EntityCallbacksListener {
+
+  @PrePersist
+  void prePersist(Object entity) {
+    EntityCallbackExecutor.create(PrePersist.class).execute(entity, entity.getClass());
+  }
+
+  @PreRemove
+  void preRemove(Object entity) {
+    EntityCallbackExecutor.create(PreRemove.class).execute(entity, entity.getClass());
+  }
+
+  @PostPersist
+  void postPersist(Object entity) {
+    EntityCallbackExecutor.create(PostPersist.class).execute(entity, entity.getClass());
+  }
+
+  @PostRemove
+  void postRemove(Object entity) {
+    EntityCallbackExecutor.create(PostRemove.class).execute(entity, entity.getClass());
+  }
+
+  @PreUpdate
+  void preUpdate(Object entity) {
+    EntityCallbackExecutor.create(PreUpdate.class).execute(entity, entity.getClass());
+  }
+
+  @PostUpdate
+  void postUpdate(Object entity) {
+    EntityCallbackExecutor.create(PostUpdate.class).execute(entity, entity.getClass());
+  }
+
+  @PostLoad
+  void postLoad(Object entity) {
+    EntityCallbackExecutor.create(PostLoad.class).execute(entity, entity.getClass());
+  }
+
+  private static class EntityCallbackExecutor {
+    Class<? extends Annotation> callbackType;
+
+    private EntityCallbackExecutor(Class<? extends Annotation> callbackType) {
+      this.callbackType = callbackType;
+    }
+
+    private static EntityCallbackExecutor create(Class<? extends Annotation> callbackType) {
+      return new EntityCallbackExecutor(callbackType);
+    }
+
+    /**
+     * Executes eligible callbacks in {@link Embedded} properties recursively.
+     *
+     * @param entity the Java object of the entity class
+     * @param entityType either the type of the entity or an ancestor type
+     */
+    private void execute(Object entity, Class<?> entityType) {
+      Class<?> parentType = entityType.getSuperclass();
+      if (parentType != null && parentType.isAnnotationPresent(MappedSuperclass.class)) {
+        execute(entity, parentType);
+      }
+
+      findEmbeddedProperties(entity, entityType)
+          .forEach(
+              normalEmbedded -> {
+                // For each normal embedded property, we don't execute its callback method because
+                // it is handled by Hibernate. However, for the embedded property defined in the
+                // entity's parent class, we need to treat it as a nested embedded property and
+                // invoke its callback function.
+                if (entity.getClass().equals(entityType)) {
+                  executeCallbackForNormalEmbeddedProperty(
+                      normalEmbedded, normalEmbedded.getClass());
+                } else {
+                  executeCallbackForNestedEmbeddedProperty(
+                      normalEmbedded, normalEmbedded.getClass());
+                }
+              });
+    }
+
+    private void executeCallbackForNestedEmbeddedProperty(
+        Object nestedEmbeddedObject, Class<?> nestedEmbeddedType) {
+      Class<?> parentType = nestedEmbeddedType.getSuperclass();
+      if (parentType != null && parentType.isAnnotationPresent(MappedSuperclass.class)) {
+        executeCallbackForNestedEmbeddedProperty(nestedEmbeddedObject, parentType);
+      }
+
+      findEmbeddedProperties(nestedEmbeddedObject, nestedEmbeddedType)
+          .forEach(
+              embeddedProperty ->
+                  executeCallbackForNestedEmbeddedProperty(
+                      embeddedProperty, embeddedProperty.getClass()));
+
+      for (Method method : nestedEmbeddedType.getDeclaredMethods()) {
+        if (method.isAnnotationPresent(callbackType)) {
+          invokeMethod(method, nestedEmbeddedObject);
+        }
+      }
+    }
+
+    private void executeCallbackForNormalEmbeddedProperty(
+        Object normalEmbeddedObject, Class<?> normalEmbeddedType) {
+      Class<?> parentType = normalEmbeddedType.getSuperclass();
+      if (parentType != null && parentType.isAnnotationPresent(MappedSuperclass.class)) {
+        executeCallbackForNormalEmbeddedProperty(normalEmbeddedObject, parentType);
+      }
+
+      findEmbeddedProperties(normalEmbeddedObject, normalEmbeddedType)
+          .forEach(
+              embeddedProperty ->
+                  executeCallbackForNestedEmbeddedProperty(
+                      embeddedProperty, embeddedProperty.getClass()));
+    }
+
+    private Stream<Object> findEmbeddedProperties(Object object, Class<?> clazz) {
+      return Arrays.stream(clazz.getDeclaredFields())
+          .filter(field -> !field.isAnnotationPresent(Transient.class))
+          .filter(
+              field ->
+                  field.isAnnotationPresent(Embedded.class)
+                      || field.getType().isAnnotationPresent(Embeddable.class))
+          .filter(field -> !Modifier.isStatic(field.getModifiers()))
+          .map(field -> getFieldObject(field, object))
+          .filter(Objects::nonNull);
+    }
+
+    private static Object getFieldObject(Field field, Object object) {
+      field.setAccessible(true);
+      try {
+        return field.get(object);
+      } catch (IllegalAccessException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    private static void invokeMethod(Method method, Object object) {
+      method.setAccessible(true);
+      try {
+        method.invoke(object);
+      } catch (IllegalAccessException | InvocationTargetException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

core/src/main/java/google/registry/persistence/NomulusPostgreSQLDialect.java

@@ -14,6 +14,7 @@
 package google.registry.persistence;
 
 import google.registry.persistence.converter.StringCollectionDescriptor;
+import google.registry.persistence.converter.StringMapDescriptor;
 import java.sql.Types;
 import org.hibernate.boot.model.TypeContributions;
 import org.hibernate.dialect.PostgreSQL95Dialect;
@@ -26,7 +27,7 @@ public class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
     registerColumnType(Types.VARCHAR, "text");
     registerColumnType(Types.TIMESTAMP_WITH_TIMEZONE, "timestamptz");
     registerColumnType(Types.TIMESTAMP, "timestamptz");
-    registerColumnType(Types.OTHER, "hstore");
+    registerColumnType(StringMapDescriptor.COLUMN_TYPE, StringMapDescriptor.COLUMN_NAME);
     registerColumnType(
         StringCollectionDescriptor.COLUMN_TYPE, StringCollectionDescriptor.COLUMN_DDL_NAME);
   }
@@ -37,5 +38,7 @@ public class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
     super.contributeTypes(typeContributions, serviceRegistry);
     typeContributions.contributeJavaTypeDescriptor(StringCollectionDescriptor.getInstance());
     typeContributions.contributeSqlTypeDescriptor(StringCollectionDescriptor.getInstance());
+    typeContributions.contributeJavaTypeDescriptor(StringMapDescriptor.getInstance());
+    typeContributions.contributeSqlTypeDescriptor(StringMapDescriptor.getInstance());
   }
 }

core/src/main/java/google/registry/persistence/VKey.java

@@ -15,6 +15,7 @@
 package google.registry.persistence;
 
 import static com.google.common.base.Preconditions.checkState;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import google.registry.model.ImmutableObject;
 import java.util.Optional;
@@ -41,20 +42,31 @@ public class VKey<T> extends ImmutableObject {
     this.primaryKey = primaryKey;
   }
 
-  public static <T> VKey<T> createSql(Class<? extends T> kind, Object primaryKey) {
-    return new VKey(kind, null, primaryKey);
+  /** Creates a {@link VKey} which only contains the sql primary key. */
+  public static <T> VKey<T> createSql(Class<? extends T> kind, Object sqlKey) {
+    checkArgumentNotNull(kind, "kind must not be null");
+    checkArgumentNotNull(sqlKey, "sqlKey must not be null");
+    return new VKey(kind, null, sqlKey);
   }
 
+  /** Creates a {@link VKey} which only contains the ofy primary key. */
   public static <T> VKey<T> createOfy(
-      Class<? extends T> kind, com.googlecode.objectify.Key<T> ofyKey) {
+      Class<? extends T> kind, com.googlecode.objectify.Key<? extends T> ofyKey) {
+    checkArgumentNotNull(kind, "kind must not be null");
+    checkArgumentNotNull(ofyKey, "ofyKey must not be null");
     return new VKey(kind, ofyKey, null);
   }
 
+  /** Creates a {@link VKey} which only contains both sql and ofy primary key. */
   public static <T> VKey<T> create(
-      Class<? extends T> kind, Object primaryKey, com.googlecode.objectify.Key ofyKey) {
-    return new VKey(kind, ofyKey, primaryKey);
+      Class<? extends T> kind, Object sqlKey, com.googlecode.objectify.Key ofyKey) {
+    checkArgumentNotNull(kind, "kind must not be null");
+    checkArgumentNotNull(sqlKey, "sqlKey must not be null");
+    checkArgumentNotNull(ofyKey, "ofyKey must not be null");
+    return new VKey(kind, ofyKey, sqlKey);
   }
 
+  /** Returns the type of the entity. */
   public Class<? extends T> getKind() {
     return this.kind;
   }

core/src/main/java/google/registry/persistence/WithLongVKey.java

@@ -0,0 +1,34 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Entity;
+
+/**
+ * Annotation for {@link Entity} which id is long type and needs an {@link AttributeConverter} for
+ * its VKey.
+ */
+@Target({ElementType.TYPE})
+public @interface WithLongVKey {
+  /**
+   * Sets the suffix of the class name for the {@link AttributeConverter} generated by
+   * LongVKeyProcessor. If not set, the suffix will be the type name of the VKey. Note that the
+   * class name will be "VKeyConverter_" concatenated with the suffix.
+   */
+  String classNameSuffix() default "";
+}