Increase the maximum number of nodes in a nood pool to 15 (#672 )

Convert RDE tests (and some test rules) from JUnit 4 to JUnit 5 (#677 )
* Add JUnit Params and start using it * Convert rest of RDE tests * Don't check headers for generated tests * Expand visibility to fix build breakage * Bump JUnit versions to 5.6.2
2026-08-07 15:56:08 +00:00 · 2020-07-10 21:54:18 -04:00 · 2020-07-10 21:32:36 -04:00 · 2020-07-10 15:57:59 -04:00 · 2020-07-10 14:56:28 -04:00 · 2020-07-10 10:03:59 -04:00
347 changed files with 7230 additions and 4333 deletions
@@ -25,7 +25,7 @@ buildscript {

  dependencies {
    classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.0.1'
-    classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.6.1"
+    classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.6.1'
    classpath 'org.sonatype.aether:aether-api:1.13.1'
    classpath 'org.sonatype.aether:aether-impl:1.13.1'
  }
@@ -49,6 +49,7 @@ plugins {
  id 'com.diffplug.gradle.spotless' version '3.25.0'

  id 'jacoco'
+  id 'com.dorongold.task-tree' version '1.5'
 }

 wrapper {
@@ -443,6 +444,7 @@ task javaIncrementalFormatDryRun {
    println("${invokeJavaDiffFormatScript("show")}")
  }
 }
+tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)

 // Checks if modified lines in Java source files need reformatting.
 // Note that this task processes modified Java files in the entire repository.
@@ -452,8 +454,6 @@ task javaIncrementalFormatApply {
  }
 }

-tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)
-
 task javadoc(type: Javadoc) {
  source javadocSource
  classpath = files(javadocClasspath)
@@ -468,4 +468,16 @@ task javadoc(type: Javadoc) {

 tasks.build.dependsOn(tasks.javadoc)

+// Task for doing development on core Nomulus.
+// This fixes code formatting automatically as necessary, builds and tests the
+// core Nomulus codebase, and runs all presubmits.
+task coreDev {
+  dependsOn 'javaIncrementalFormatApply'
+  dependsOn 'javadoc'
+  dependsOn 'checkDependenciesDotGradle'
+  dependsOn 'checkLicense'
+  dependsOn ':core:check'
+  dependsOn 'assemble'
+}
+
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
@@ -50,8 +50,8 @@ javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
-net.bytebuddy:byte-buddy:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
+net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.8
@@ -61,13 +61,13 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
-org.mockito:mockito-core:2.25.0
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
+org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm-analysis:6.0
@@ -50,8 +50,8 @@ javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
-net.bytebuddy:byte-buddy:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
+net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.8
@@ -61,13 +61,13 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
-org.mockito:mockito-core:2.25.0
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
+org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm-analysis:6.0
@@ -50,8 +50,8 @@ javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
-net.bytebuddy:byte-buddy:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
+net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.8
@@ -61,13 +61,13 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
-org.mockito:mockito-core:2.25.0
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
+org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm-analysis:6.0
@@ -50,8 +50,8 @@ javax.inject:javax.inject:1
 javax.validation:validation-api:1.0.0.GA
 joda-time:joda-time:2.9.2
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
-net.bytebuddy:byte-buddy:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
+net.bytebuddy:byte-buddy:1.10.5
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.8
@@ -61,13 +61,13 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.hamcrest:hamcrest-core:1.3
 org.json:json:20160212
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
-org.mockito:mockito-core:2.25.0
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
+org.mockito:mockito-core:3.3.3
 org.objenesis:objenesis:2.6
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm-analysis:6.0
@@ -29,6 +29,7 @@ import google.registry.gradle.plugin.ProjectData.TaskData;
 import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Map;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;

@@ -162,7 +163,7 @@ final class CoverPageGenerator {
            task.reports().entrySet().stream()
                .collect(
                    toImmutableMap(
-                        entry -> entry.getKey(),
+                        Map.Entry::getKey,
                        entry ->
                            entry.getValue().files().isEmpty()
                                ? ""
@@ -168,7 +168,7 @@ final class GcsPluginUtils {
            .collect(
                toImmutableMap(
                    file -> rootDir.relativize(toNormalizedPath(file)),
-                    file -> toByteArraySupplier(file)));
+                    GcsPluginUtils::toByteArraySupplier));

    if (files.isEmpty()) {
      // The directory exists, but is empty. Return empty FilesWithEntryPoint
@@ -20,10 +20,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0
@@ -20,10 +20,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0
@@ -21,10 +21,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0
@@ -21,10 +21,10 @@ org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
 org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r
 org.hamcrest:hamcrest-core:1.3
-org.junit.jupiter:junit-jupiter-api:5.6.1
-org.junit.jupiter:junit-jupiter-engine:5.6.1
-org.junit.platform:junit-platform-commons:1.6.1
-org.junit.platform:junit-platform-engine:1.6.1
-org.junit.vintage:junit-vintage-engine:5.6.1
-org.junit:junit-bom:5.6.1
+org.junit.jupiter:junit-jupiter-api:5.6.2
+org.junit.jupiter:junit-jupiter-engine:5.6.2
+org.junit.platform:junit-platform-commons:1.6.2
+org.junit.platform:junit-platform-engine:1.6.2
+org.junit.vintage:junit-vintage-engine:5.6.2
+org.junit:junit-bom:5.6.2
 org.opentest4j:opentest4j:1.2.0
@@ -77,9 +77,9 @@ PRESUBMITS = {
    PresubmitCheck(
        r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
        ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
-            ".git", "/build/", "/generated/", "node_modules/",
-            "JUnitBackports.java", "registrar_bin.", "registrar_dbg.",
-            "google-java-format-diff.py",
+            ".git", "/build/", "/generated/", "/generated_tests/",
+            "node_modules/", "JUnitBackports.java", "registrar_bin.",
+            "registrar_dbg.", "google-java-format-diff.py",
            "nomulus.golden.sql", "soyutils_usegoog.js"
        }, REQUIRED):
        "File did not include the license header.",
@@ -312,10 +312,12 @@ dependencies {
  testCompile deps['org.junit.jupiter:junit-jupiter-api']
  testCompile deps['org.junit.jupiter:junit-jupiter-engine']
  testCompile deps['org.junit.jupiter:junit-jupiter-migrationsupport']
+  testCompile deps['org.junit.jupiter:junit-jupiter-params']
  testCompile deps['org.junit.platform:junit-platform-runner']
  testCompile deps['org.junit.platform:junit-platform-suite-api']
  testCompile deps['org.junit.vintage:junit-vintage-engine']
  testCompile deps['org.mockito:mockito-core']
+  testCompile deps['org.mockito:mockito-junit-jupiter']
  runtime deps['org.postgresql:postgresql']

  // Indirect dependency found by undeclared-dependency check. Such
@@ -186,7 +186,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.3
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -246,6 +246,7 @@ org.json:json:20160810
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
@@ -254,7 +255,8 @@ org.junit.platform:junit-platform-suite-api:1.6.2
 org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
-org.mockito:mockito-core:2.25.0
+org.mockito:mockito-core:3.3.3
+org.mockito:mockito-junit-jupiter:3.3.3
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
 org.objenesis:objenesis:2.6
@@ -184,7 +184,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.3
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -244,6 +244,7 @@ org.json:json:20160810
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
@@ -252,7 +253,8 @@ org.junit.platform:junit-platform-suite-api:1.6.2
 org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
-org.mockito:mockito-core:2.25.0
+org.mockito:mockito-core:3.3.3
+org.mockito:mockito-junit-jupiter:3.3.3
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
 org.objenesis:objenesis:2.6
@@ -188,7 +188,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.3
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -249,6 +249,7 @@ org.json:json:20160810
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
@@ -257,7 +258,8 @@ org.junit.platform:junit-platform-suite-api:1.6.2
 org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
-org.mockito:mockito-core:2.25.0
+org.mockito:mockito-core:3.3.3
+org.mockito:mockito-junit-jupiter:3.3.3
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
 org.objenesis:objenesis:2.6
@@ -188,7 +188,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.3
 junit:junit:4.13
-net.bytebuddy:byte-buddy-agent:1.9.7
+net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -249,6 +249,7 @@ org.json:json:20160810
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
 org.junit.jupiter:junit-jupiter-migrationsupport:5.6.2
+org.junit.jupiter:junit-jupiter-params:5.6.2
 org.junit.platform:junit-platform-commons:1.6.2
 org.junit.platform:junit-platform-engine:1.6.2
 org.junit.platform:junit-platform-launcher:1.6.2
@@ -257,7 +258,8 @@ org.junit.platform:junit-platform-suite-api:1.6.2
 org.junit.vintage:junit-vintage-engine:5.6.2
 org.junit:junit-bom:5.6.2
 org.jvnet.staxex:stax-ex:1.8
-org.mockito:mockito-core:2.25.0
+org.mockito:mockito-core:3.3.3
+org.mockito:mockito-junit-jupiter:3.3.3
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
 org.objenesis:objenesis:2.6
@@ -23,7 +23,8 @@ import java.lang.reflect.Proxy;

 /**
 * Sets up a placeholder {@link Environment} on a non-AppEngine platform so that Datastore Entities
- * can be deserialized. See {@code DatastoreEntityExtension} in test source for more information.
+ * can be converted from/to Objectify entities. See {@code DatastoreEntityExtension} in test source
+ * for more information.
 */
 public class AppEngineEnvironment implements Closeable {

@@ -285,11 +285,9 @@ public class DeleteContactsAndHostsAction implements Runnable {
      if (resourceKey.getKind().equals(KIND_CONTACT)) {
        return domain
            .getReferencedContacts()
-            .contains(VKey.createOfy(ContactResource.class, (Key<ContactResource>) resourceKey));
+            .contains(VKey.from((Key<ContactResource>) resourceKey));
      } else if (resourceKey.getKind().equals(KIND_HOST)) {
-        return domain
-            .getNameservers()
-            .contains(VKey.createOfy(HostResource.class, (Key<HostResource>) resourceKey));
+        return domain.getNameservers().contains(VKey.from((Key<HostResource>) resourceKey));
      } else {
        throw new IllegalStateException("EPP resource key of unknown type: " + resourceKey);
      }
@@ -207,9 +207,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
      Key<HostResource> referencingHostKey = null;
      for (DnsRefreshRequest request : refreshRequests) {
        if (isActive(domain, request.lastUpdateTime())
-            && domain
-                .getNameservers()
-                .contains(VKey.createOfy(HostResource.class, request.hostKey()))) {
+            && domain.getNameservers().contains(VKey.from(request.hostKey()))) {
          referencingHostKey = request.hostKey();
          break;
        }
@@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Strings.isNullOrEmpty;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import org.joda.time.DateTime;

@@ -36,6 +37,23 @@ public final class BackupPaths {
  public static final String COMMIT_LOG_NAME_PREFIX = "commit_diff_until_";
  private static final String COMMIT_LOG_PATTERN_TEMPLATE = "%s/" + COMMIT_LOG_NAME_PREFIX + "*";

+  /**
+   * Pattern of the per-project file with Cloud SQL connection information. To get a concrete path,
+   * user needs to provide the name of the environment, alpha, crash, sandbox, or production. This
+   * file is meant for applications without access to secrets stored in Datastore.
+   *
+   * <p>In production, this is an base-64 encoded encrypted file with one line, which contains
+   * space-separated values of Cloud SQL instance name, login, and password.
+   *
+   * <p>A plain text may be used for tests to a local database. Replace Cloud SQL instance name with
+   * JDBC URL.
+   */
+  private static final String SQL_CONN_INFO_FILE_PATTERN =
+      "gs://domain-registry-dev-deploy/cloudsql-credentials/%s/admin_credential.enc";
+
+  private static final ImmutableSet<String> ALLOWED_ENV =
+      ImmutableSet.of("alpha", "crash", "sandbox", "production");
+
  /**
   * Returns a regex pattern that matches all Datastore export files of a given {@code kind}.
   *
@@ -90,4 +108,10 @@ public final class BackupPaths {
    checkArgument(start >= 0, "Illegal file name %s.", fileName);
    return DateTime.parse(fileName.substring(start + COMMIT_LOG_NAME_PREFIX.length()));
  }
+
+  public static ImmutableList<String> getCloudSQLCredentialFilePatterns(String environmentName) {
+    checkArgument(
+        ALLOWED_ENV.contains(environmentName), "Invalid environment name %s", environmentName);
+    return ImmutableList.of(String.format(SQL_CONN_INFO_FILE_PATTERN, environmentName));
+  }
 }
@@ -0,0 +1,192 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import com.google.common.base.Splitter;
+import dagger.Binds;
+import dagger.Component;
+import dagger.Lazy;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.keyring.kms.KmsModule;
+import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.JdbcJpaTm;
+import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.util.Clock;
+import google.registry.util.Sleeper;
+import google.registry.util.SystemClock;
+import google.registry.util.SystemSleeper;
+import google.registry.util.UtilsModule;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.channels.Channels;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import javax.annotation.Nullable;
+import javax.inject.Singleton;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResourceId;
+
+/**
+ * Provides bindings for {@link JpaTransactionManager} to Cloud SQL.
+ *
+ * <p>This module is intended for use in BEAM pipelines, and uses a BEAM utility to access GCS like
+ * a regular file system.
+ */
+@Module
+public class BeamJpaModule {
+
+  private static final String GCS_SCHEME = "gs://";
+
+  @Nullable private final String credentialFilePath;
+
+  /**
+   * Constructs a new instance of {@link BeamJpaModule}.
+   *
+   * <p>Note: it is an unfortunately necessary antipattern to check for the validity of
+   * credentialFilePath in {@link #provideCloudSqlAccessInfo} rather than in the constructor.
+   * Unfortunately, this is a restriction imposed upon us by Dagger. Specifically, because we use
+   * this in at least one 1 {@link google.registry.tools.RegistryTool} command(s), it must be
+   * instantiated in {@code google.registry.tools.RegistryToolComponent} for all possible commands;
+   * Dagger doesn't permit it to ever be null. For the vast majority of commands, it will never be
+   * used (so a null credential file path is fine in those cases).
+   *
+   * @param credentialFilePath the path to a Cloud SQL credential file. This must refer to either a
+   *     real encrypted file on GCS as returned by {@link
+   *     BackupPaths#getCloudSQLCredentialFilePatterns} or an unencrypted file on local filesystem
+   *     with credentials to a test database.
+   */
+  public BeamJpaModule(@Nullable String credentialFilePath) {
+    this.credentialFilePath = credentialFilePath;
+  }
+
+  /** Returns true if the credential file is on GCS (and therefore expected to be encrypted). */
+  private boolean isCloudSqlCredential() {
+    return credentialFilePath.startsWith(GCS_SCHEME);
+  }
+
+  @Provides
+  @Singleton
+  SqlAccessInfo provideCloudSqlAccessInfo(Lazy<CloudSqlCredentialDecryptor> lazyDecryptor) {
+    checkArgument(!isNullOrEmpty(credentialFilePath), "Null or empty credentialFilePath");
+    String line = readOnlyLineFromCredentialFile();
+    if (isCloudSqlCredential()) {
+      line = lazyDecryptor.get().decrypt(line);
+    }
+    // See ./BackupPaths.java for explanation of the line format.
+    List<String> parts = Splitter.on(' ').splitToList(line.trim());
+    checkState(parts.size() == 3, "Expecting three phrases in %s", line);
+    if (isCloudSqlCredential()) {
+      return SqlAccessInfo.createCloudSqlAccessInfo(parts.get(0), parts.get(1), parts.get(2));
+    } else {
+      return SqlAccessInfo.createLocalSqlAccessInfo(parts.get(0), parts.get(1), parts.get(2));
+    }
+  }
+
+  String readOnlyLineFromCredentialFile() {
+    try {
+      ResourceId resourceId = FileSystems.matchSingleFileSpec(credentialFilePath).resourceId();
+      try (BufferedReader reader =
+          new BufferedReader(
+              new InputStreamReader(
+                  Channels.newInputStream(FileSystems.open(resourceId)), StandardCharsets.UTF_8))) {
+        return reader.readLine();
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Provides
+  @Config("beamCloudSqlJdbcUrl")
+  String provideJdbcUrl(SqlAccessInfo sqlAccessInfo) {
+    return sqlAccessInfo.jdbcUrl();
+  }
+
+  @Provides
+  @Config("beamCloudSqlInstanceConnectionName")
+  String provideSqlInstanceName(SqlAccessInfo sqlAccessInfo) {
+    return sqlAccessInfo
+        .cloudSqlInstanceName()
+        .orElseThrow(() -> new IllegalStateException("Cloud SQL not provisioned."));
+  }
+
+  @Provides
+  @Config("beamCloudSqlUsername")
+  String provideSqlUsername(SqlAccessInfo sqlAccessInfo) {
+    return sqlAccessInfo.user();
+  }
+
+  @Provides
+  @Config("beamCloudSqlPassword")
+  String provideSqlPassword(SqlAccessInfo sqlAccessInfo) {
+    return sqlAccessInfo.password();
+  }
+
+  @Provides
+  @Config("beamCloudKmsProjectId")
+  static String kmsProjectId() {
+    return "domain-registry-dev";
+  }
+
+  @Provides
+  @Config("beamCloudKmsKeyRing")
+  static String keyRingName() {
+    return "nomulus-tool-keyring";
+  }
+
+  @Provides
+  @Config("beamHibernateHikariMaximumPoolSize")
+  static int getBeamHibernateHikariMaximumPoolSize() {
+    return 4;
+  }
+
+  @Module
+  interface BindModule {
+
+    @Binds
+    Sleeper sleeper(SystemSleeper sleeper);
+
+    @Binds
+    Clock clock(SystemClock clock);
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        ConfigModule.class,
+        CredentialModule.class,
+        BeamJpaModule.class,
+        KmsModule.class,
+        PersistenceModule.class,
+        UtilsModule.class
+      })
+  public interface JpaTransactionManagerComponent {
+    @SocketFactoryJpaTm
+    JpaTransactionManager cloudSqlJpaTransactionManager();
+
+    @JdbcJpaTm
+    JpaTransactionManager localDbJpaTransactionManager();
+  }
+}
@@ -0,0 +1,50 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.api.services.cloudkms.v1.model.DecryptRequest;
+import com.google.common.base.Strings;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.keyring.kms.KmsConnection;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import javax.inject.Inject;
+
+/**
+ * Decrypts data using Cloud KMS, with the same crypto key with which Cloud SQL credential files on
+ * GCS was encrypted. See {@link BackupPaths#getCloudSQLCredentialFilePatterns} for more
+ * information.
+ */
+public class CloudSqlCredentialDecryptor {
+
+  private static final String CRYPTO_KEY_NAME = "nomulus-tool-key";
+  private final KmsConnection kmsConnection;
+
+  @Inject
+  CloudSqlCredentialDecryptor(@Config("beamKmsConnection") KmsConnection kmsConnection) {
+    this.kmsConnection = kmsConnection;
+  }
+
+  public String decrypt(String data) {
+    checkArgument(!Strings.isNullOrEmpty(data), "Null or empty data.");
+    byte[] ciphertext = Base64.getDecoder().decode(data);
+    // Re-encode for Cloud KMS JSON REST API, invoked through kmsConnection.
+    String urlSafeCipherText = new DecryptRequest().encodeCiphertext(ciphertext).getCiphertext();
+    return new String(
+        kmsConnection.decrypt(CRYPTO_KEY_NAME, urlSafeCipherText), StandardCharsets.UTF_8);
+  }
+}
@@ -0,0 +1,45 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import com.google.auto.value.AutoValue;
+import java.util.Optional;
+
+/**
+ * Information needed to connect to a database, including JDBC URL, user name, password, and in the
+ * case of Cloud SQL, the database instance's name.
+ */
+@AutoValue
+abstract class SqlAccessInfo {
+
+  abstract String jdbcUrl();
+
+  abstract String user();
+
+  abstract String password();
+
+  abstract Optional<String> cloudSqlInstanceName();
+
+  public static SqlAccessInfo createCloudSqlAccessInfo(
+      String sqlInstanceName, String username, String password) {
+    return new AutoValue_SqlAccessInfo(
+        "jdbc:postgresql://google/postgres", username, password, Optional.of(sqlInstanceName));
+  }
+
+  public static SqlAccessInfo createLocalSqlAccessInfo(
+      String jdbcUrl, String username, String password) {
+    return new AutoValue_SqlAccessInfo(jdbcUrl, username, password, Optional.empty());
+  }
+}
@@ -16,18 +16,27 @@ package google.registry.beam.initsql;

 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static java.util.Comparator.comparing;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;

 import avro.shaded.com.google.common.collect.Iterators;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
 import google.registry.backup.CommitLogImports;
 import google.registry.backup.VersionedEntity;
 import google.registry.tools.LevelDbLogReader;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.Optional;
+import java.util.Set;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.Compression;
 import org.apache.beam.sdk.io.FileIO;
@@ -36,11 +45,20 @@ import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ProcessFunction;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.TypeDescriptor;
 import org.joda.time.DateTime;

 /**
@@ -57,6 +75,110 @@ public final class Transforms {
   */
  @VisibleForTesting static final long EXPORT_ENTITY_TIME_STAMP = START_OF_TIME.getMillis();

+  /**
+   * Returns a {@link TupleTag} that can be used to retrieve entities of the given {@code kind} from
+   * the Datastore snapshot returned by {@link #loadDatastoreSnapshot}.
+   */
+  public static TupleTag<VersionedEntity> createTagForKind(String kind) {
+    // When used with PCollectionTuple the result must retain generic type information.
+    // Both the Generic param and the empty bracket below are important.
+    return new TupleTag<VersionedEntity>(Transforms.class.getSimpleName() + ":" + kind) {};
+  }
+
+  /**
+   * Composite {@link PTransform transform} that loads the Datastore snapshot at {@code
+   * commitLogToTime} for caller specified {@code kinds}.
+   *
+   * <p>Caller must provide the location of a Datastore export that started AFTER {@code
+   * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory
+   * of all CommitLog files.
+   *
+   * <p>Selection of {@code commitLogFromTime} and {@code commitLogToTime} should follow the
+   * guidelines below to ensure that all incremental changes concurrent with the export are covered:
+   *
+   * <ul>
+   *   <li>Two or more CommitLogs should exist between {@code commitLogFromTime} and the starting
+   *       time of the Datastore export. This ensures that the earlier CommitLog file was complete
+   *       before the export started.
+   *   <li>Two or more CommitLogs should exit between the export completion time and {@code
+   *       commitLogToTime}.
+   * </ul>
+   *
+   * <p>The output from the returned transform is a {@link PCollectionTuple} consisting of {@link
+   * VersionedEntity VersionedEntities} grouped into {@link PCollection PCollections} by {@code
+   * kind}.
+   */
+  public static PTransform<PBegin, PCollectionTuple> loadDatastoreSnapshot(
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<String> kinds) {
+    checkArgument(kinds != null && !kinds.isEmpty(), "At least one kind is expected.");
+
+    // Create tags to collect entities by kind in final step.
+    final ImmutableMap<String, TupleTag<VersionedEntity>> outputTags =
+        kinds.stream()
+            .collect(ImmutableMap.toImmutableMap(kind -> kind, Transforms::createTagForKind));
+    // Arbitrarily select one tag as mainOutTag and put the remaining ones in a TupleTagList.
+    // This separation is required by ParDo's config API.
+    Iterator<TupleTag<VersionedEntity>> tagsIt = outputTags.values().iterator();
+    final TupleTag<VersionedEntity> mainOutputTag = tagsIt.next();
+    final TupleTagList additionalTags = TupleTagList.of(ImmutableList.copyOf(tagsIt));
+
+    return new PTransform<PBegin, PCollectionTuple>() {
+      @Override
+      public PCollectionTuple expand(PBegin input) {
+        PCollection<VersionedEntity> exportedEntities =
+            input
+                .apply("Get export file patterns", getDatastoreExportFilePatterns(exportDir, kinds))
+                .apply("Find export files", getFilesByPatterns())
+                .apply("Load export data", loadExportDataFromFiles());
+        PCollection<VersionedEntity> commitLogEntities =
+            input
+                .apply("Get commitlog file patterns", getCommitLogFilePatterns(commitLogDir))
+                .apply("Find commitlog files", getFilesByPatterns())
+                .apply(
+                    "Filter commitLog by time",
+                    filterCommitLogsByTime(commitLogFromTime, commitLogToTime))
+                .apply("Load commitlog data", loadCommitLogsFromFiles(kinds));
+        return PCollectionList.of(exportedEntities)
+            .and(commitLogEntities)
+            .apply("Merge exports and CommitLogs", Flatten.pCollections())
+            .apply(
+                "Key entities by Datastore Keys",
+                // Converting to KV<String, VE> instead of KV<Key, VE> b/c default coder for Key
+                // (SerializableCoder) is not deterministic and cannot be used with GroupBy.
+                MapElements.into(kvs(strings(), TypeDescriptor.of(VersionedEntity.class)))
+                    .via((VersionedEntity e) -> KV.of(e.key().toString(), e)))
+            .apply("Gather entities by key", GroupByKey.create())
+            .apply(
+                "Output latest version per entity",
+                ParDo.of(
+                        new DoFn<KV<String, Iterable<VersionedEntity>>, VersionedEntity>() {
+                          @ProcessElement
+                          public void processElement(
+                              @Element KV<String, Iterable<VersionedEntity>> kv,
+                              MultiOutputReceiver out) {
+                            Optional<VersionedEntity> latest =
+                                Streams.stream(kv.getValue())
+                                    .sorted(comparing(VersionedEntity::commitTimeMills).reversed())
+                                    .findFirst();
+                            // Throw to abort (after default retries). Investigate, fix, and rerun.
+                            checkState(
+                                latest.isPresent(), "Unexpected key with no data", kv.getKey());
+                            if (latest.get().isDelete()) {
+                              return;
+                            }
+                            String kind = latest.get().getEntity().get().getKind();
+                            out.get(outputTags.get(kind)).output(latest.get());
+                          }
+                        })
+                    .withOutputTags(mainOutputTag, additionalTags));
+      }
+    };
+  }
+
  /**
   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
   * all Datastore CommitLog files.
@@ -75,6 +197,11 @@ public final class Transforms {
    return toStringPCollection(getExportFilePatterns(exportDir, kinds));
  }

+  public static PTransform<PBegin, PCollection<String>> getCloudSqlConnectionInfoFilePatterns(
+      String gcpProjectName) {
+    return toStringPCollection(BackupPaths.getCloudSQLCredentialFilePatterns(gcpProjectName));
+  }
+
  /**
   * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}.
   */
@@ -91,7 +218,7 @@ public final class Transforms {
   * Returns CommitLog files with timestamps between {@code fromTime} (inclusive) and {@code
   * endTime} (exclusive).
   */
-  public static PTransform<PCollection<? extends String>, PCollection<String>>
+  public static PTransform<PCollection<? extends Metadata>, PCollection<Metadata>>
      filterCommitLogsByTime(DateTime fromTime, DateTime toTime) {
    return ParDo.of(new FilterCommitLogFileByTime(fromTime, toTime));
  }
@@ -109,9 +236,13 @@ public final class Transforms {

  /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
-      loadCommitLogsFromFiles() {
+      loadCommitLogsFromFiles(Set<String> kinds) {
    return processFiles(
-        new BackupFileReader(file -> CommitLogImports.loadEntities(file.open()).iterator()));
+        new BackupFileReader(
+            file ->
+                CommitLogImports.loadEntities(file.open()).stream()
+                    .filter(e -> kinds.contains(e.key().getKind()))
+                    .iterator()));
  }

  /**
@@ -134,12 +265,11 @@ public final class Transforms {
        return input
            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
            .apply(transformer.getClass().getSimpleName(), ParDo.of(transformer));
-        // TODO(weiminyu): reshuffle to enable dynamic work rebalance per beam dev guide
      }
    };
  }

-  private static class FilterCommitLogFileByTime extends DoFn<String, String> {
+  private static class FilterCommitLogFileByTime extends DoFn<Metadata, Metadata> {
    private final DateTime fromTime;
    private final DateTime toTime;

@@ -156,10 +286,10 @@ public final class Transforms {
    }

    @ProcessElement
-    public void processElement(@Element String fileName, OutputReceiver<String> out) {
-      DateTime timestamp = getCommitLogTimestamp(fileName);
+    public void processElement(@Element Metadata fileMeta, OutputReceiver<Metadata> out) {
+      DateTime timestamp = getCommitLogTimestamp(fileMeta.resourceId().toString());
      if (isBeforeOrAt(fromTime, timestamp) && timestamp.isBefore(toTime)) {
-        out.output(fileName);
+        out.output(fileMeta);
      }
    }
  }
@@ -235,6 +235,7 @@ public abstract class BillingEvent implements Serializable {
                DATE_TIME_FORMATTER.format(eventTime()),
                registrarId(),
                billingId(),
+                poNumber(),
                tld(),
                action(),
                domain(),
@@ -56,7 +56,8 @@ FROM (
      `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
    WHERE
    -- TODO(b/18092292): Add a filter for tldState (not PDT/PREDELEGATION)
-      tldType = 'REAL') ) AS BillingEvent
+      tldType = 'REAL'
+    AND disableInvoicing is not TRUE) ) AS BillingEvent
  -- Gather billing ID from registrar table
  -- This is a 'JOIN' as opposed to 'LEFT JOIN' to filter out
  -- non-billable registrars
@@ -141,7 +141,7 @@ public class SafeBrowsingTransforms {
    @ProcessElement
    public void processElement(ProcessContext context) {
      Subdomain subdomain = context.element();
-      subdomainBuffer.put(subdomain.fullyQualifiedDomainName(), subdomain);
+      subdomainBuffer.put(subdomain.domainName(), subdomain);
      if (subdomainBuffer.size() >= BATCH_SIZE) {
        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
        results.forEach(context::output);
@@ -239,7 +239,7 @@ public class SafeBrowsingTransforms {
            String url = match.getJSONObject("threat").getString("url");
            Subdomain subdomain = subdomainBuffer.get(url);
            resultBuilder.add(
-                KV.of(subdomain, ThreatMatch.create(match, subdomain.fullyQualifiedDomainName())));
+                KV.of(subdomain, ThreatMatch.create(match, subdomain.domainName())));
          }
        }
      }
@@ -77,7 +77,7 @@ public class Spec11Pipeline implements Serializable {
  public static final String REGISTRAR_EMAIL_FIELD = "registrarEmailAddress";
  /** The JSON object field into which we put the registrar's name for Spec11 reports. */
  public static final String REGISTRAR_CLIENT_ID_FIELD = "registrarClientId";
-  /** The JSON object field we put the threat match array for Spec11 reports. */
+  /** The JSON object field into which we put the threat match array for Spec11 reports. */
  public static final String THREAT_MATCHES_FIELD = "threatMatches";

  private final String projectId;
@@ -94,8 +94,7 @@ public class Spec11Pipeline implements Serializable {
      @Config("spec11TemplateUrl") String spec11TemplateUrl,
      @Config("reportingBucketUrl") String reportingBucketUrl,
      @LocalCredential GoogleCredentialsBundle googleCredentialsBundle,
-      Retrier retrier
-  ) {
+      Retrier retrier) {
    this.projectId = projectId;
    this.beamStagingUrl = beamStagingUrl;
    this.spec11TemplateUrl = spec11TemplateUrl;
@@ -177,9 +176,11 @@ public class Spec11Pipeline implements Serializable {
      PCollection<Subdomain> domains,
      EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
      ValueProvider<String> dateProvider) {
-    PCollection<KV<Subdomain, ThreatMatch>> subdomains =
+
+    /* Store ThreatMatch objects in JSON. */
+    PCollection<KV<Subdomain, ThreatMatch>> subdomainsJson =
        domains.apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn));
-    subdomains
+    subdomainsJson
        .apply(
            "Map registrar client ID to email/ThreatMatch pair",
            MapElements.into(
@@ -188,7 +189,7 @@ public class Spec11Pipeline implements Serializable {
                .via(
                    (KV<Subdomain, ThreatMatch> kv) ->
                        KV.of(
-                            kv.getKey().registrarClientId(),
+                            kv.getKey().registrarId(),
                            EmailAndThreatMatch.create(
                                kv.getKey().registrarEmailAddress(), kv.getValue()))))
        .apply("Group by registrar client ID", GroupByKey.create())
@@ -36,12 +36,14 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 public abstract class Subdomain implements Serializable {

  private static final ImmutableList<String> FIELD_NAMES =
-      ImmutableList.of("fullyQualifiedDomainName", "registrarClientId", "registrarEmailAddress");
+      ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress");

  /** Returns the fully qualified domain name. */
-  abstract String fullyQualifiedDomainName();
-  /** Returns the client ID of the associated registrar for this domain. */
-  abstract String registrarClientId();
+  abstract String domainName();
+  /** Returns the domain repo ID (the primary key of the domain table). */
+  abstract String domainRepoId();
+  /** Returns the registrar ID of the associated registrar for this domain. */
+  abstract String registrarId();
  /** Returns the email address of the registrar associated with this domain. */
  abstract String registrarEmailAddress();

@@ -56,8 +58,9 @@ public abstract class Subdomain implements Serializable {
    checkFieldsNotNull(FIELD_NAMES, schemaAndRecord);
    GenericRecord record = schemaAndRecord.getRecord();
    return create(
-        extractField(record, "fullyQualifiedDomainName"),
-        extractField(record, "registrarClientId"),
+        extractField(record, "domainName"),
+        extractField(record, "domainRepoId"),
+        extractField(record, "registrarId"),
        extractField(record, "registrarEmailAddress"));
  }

@@ -69,9 +72,11 @@ public abstract class Subdomain implements Serializable {
   */
  @VisibleForTesting
  static Subdomain create(
-      String fullyQualifiedDomainName, String registrarClientId, String registrarEmailAddress) {
+      String domainName,
+      String domainRepoId,
+      String registrarId,
+      String registrarEmailAddress) {
    return new AutoValue_Subdomain(
-        fullyQualifiedDomainName, registrarClientId, registrarEmailAddress);
+        domainName, domainRepoId, registrarId, registrarEmailAddress);
  }
 }
-
@@ -19,11 +19,13 @@
  -- email address.

 SELECT
-  domain.fullyQualifiedDomainName AS fullyQualifiedDomainName,
-  registrar.clientId AS registrarClientId,
+  domain.fullyQualifiedDomainName AS domainName,
+  domain.__key__.name AS domainRepoId,
+  registrar.clientId AS clientId,
  COALESCE(registrar.emailAddress, '') AS registrarEmailAddress
 FROM ( (
    SELECT
+      __key__,
      fullyQualifiedDomainName,
      currentSponsorClientId,
      creationTime
@@ -7,7 +7,7 @@
  <sessions-enabled>true</sessions-enabled>
  <instance-class>B4_1G</instance-class>
  <manual-scaling>
-    <instances>10</instances>
+    <instances>20</instances>
  </manual-scaling>

  <system-properties>
@@ -19,6 +19,7 @@ import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.model.EppResourceUtils.checkResourcesExist;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.flows.EppException;
 import google.registry.flows.ExtensionManager;
@@ -33,8 +34,6 @@ import google.registry.model.eppoutput.CheckData.ContactCheckData;
 import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.util.Clock;
-import java.util.List;
-import java.util.Set;
 import javax.inject.Inject;

 /**
@@ -59,9 +58,10 @@ public final class ContactCheckFlow implements Flow {
  public final EppResponse run() throws EppException {
    extensionManager.validate();  // There are no legal extensions for this flow.
    validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
+    ImmutableList<String> targetIds = ((Check) resourceCommand).getTargetIds();
    verifyTargetIdCount(targetIds, maxChecks);
-    Set<String> existingIds = checkResourcesExist(ContactResource.class, targetIds, clock.nowUtc());
+    ImmutableSet<String> existingIds =
+        checkResourcesExist(ContactResource.class, targetIds, clock.nowUtc());
    ImmutableList.Builder<ContactCheck> checks = new ImmutableList.Builder<>();
    for (String id : targetIds) {
      boolean unused = !existingIds.contains(id);
@@ -15,6 +15,8 @@
 package google.registry.flows.domain;

 import static com.google.common.base.Strings.emptyToNull;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.flows.domain.DomainFlowUtils.checkAllowedAccessToTld;
@@ -26,7 +28,6 @@ import static google.registry.flows.domain.DomainFlowUtils.isValidReservedCreate
 import static google.registry.flows.domain.DomainFlowUtils.validateDomainName;
 import static google.registry.flows.domain.DomainFlowUtils.validateDomainNameWithIdnTables;
 import static google.registry.flows.domain.DomainFlowUtils.verifyNotInPredelegation;
-import static google.registry.model.EppResourceUtils.checkResourcesExist;
 import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
 import static google.registry.model.registry.label.ReservationType.getTypeOfHighestSeverity;

@@ -48,11 +49,14 @@ import google.registry.flows.custom.DomainCheckFlowCustomLogic.BeforeResponsePar
 import google.registry.flows.custom.DomainCheckFlowCustomLogic.BeforeResponseReturnData;
 import google.registry.flows.domain.token.AllocationTokenDomainCheckResults;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils;
+import google.registry.model.EppResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainCommand.Check;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
 import google.registry.model.domain.fee.FeeCheckCommandExtensionItem;
 import google.registry.model.domain.fee.FeeCheckResponseExtensionItem;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
+import google.registry.model.domain.fee06.FeeCheckCommandExtensionV06;
 import google.registry.model.domain.launch.LaunchCheckExtension;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationTokenExtension;
@@ -62,14 +66,14 @@ import google.registry.model.eppoutput.CheckData.DomainCheck;
 import google.registry.model.eppoutput.CheckData.DomainCheckData;
 import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.eppoutput.EppResponse.ResponseExtension;
+import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.model.registry.label.ReservationType;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
+import google.registry.persistence.VKey;
 import google.registry.util.Clock;
 import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import javax.inject.Inject;
@@ -110,14 +114,20 @@ public final class DomainCheckFlow implements Flow {
  @Inject ExtensionManager extensionManager;
  @Inject EppInput eppInput;
  @Inject @ClientId String clientId;
-  @Inject @Config("maxChecks") int maxChecks;
+
+  @Inject
+  @Config("maxChecks")
+  int maxChecks;
+
  @Inject @Superuser boolean isSuperuser;
  @Inject Clock clock;
  @Inject EppResponse.Builder responseBuilder;
  @Inject AllocationTokenFlowUtils allocationTokenFlowUtils;
  @Inject DomainCheckFlowCustomLogic flowCustomLogic;
  @Inject DomainPricingLogic pricingLogic;
-  @Inject DomainCheckFlow() {}
+
+  @Inject
+  DomainCheckFlow() {}

  @Override
  public EppResponse run() throws EppException {
@@ -126,39 +136,41 @@ public final class DomainCheckFlow implements Flow {
    flowCustomLogic.beforeValidation();
    extensionManager.validate();
    validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
+    ImmutableList<String> domainNames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(domainNames, maxChecks);
    DateTime now = clock.nowUtc();
-    ImmutableMap.Builder<String, InternetDomainName> domains = new ImmutableMap.Builder<>();
+    ImmutableMap.Builder<String, InternetDomainName> parsedDomainsBuilder =
+        new ImmutableMap.Builder<>();
    // Only check that the registrar has access to a TLD the first time it is encountered
    Set<String> seenTlds = new HashSet<>();
-    for (String targetId : ImmutableSet.copyOf(targetIds)) {
-      InternetDomainName domainName = validateDomainName(targetId);
-      validateDomainNameWithIdnTables(domainName);
+    for (String domainName : ImmutableSet.copyOf(domainNames)) {
+      InternetDomainName parsedDomain = validateDomainName(domainName);
+      validateDomainNameWithIdnTables(parsedDomain);
      // This validation is moderately expensive, so cache the results.
-      domains.put(targetId, domainName);
-      String tld = domainName.parent().toString();
+      parsedDomainsBuilder.put(domainName, parsedDomain);
+      String tld = parsedDomain.parent().toString();
      boolean tldFirstTimeSeen = seenTlds.add(tld);
      if (tldFirstTimeSeen && !isSuperuser) {
        checkAllowedAccessToTld(clientId, tld);
        verifyNotInPredelegation(Registry.get(tld), now);
      }
    }
-    ImmutableMap<String, InternetDomainName> domainNames = domains.build();
+    ImmutableMap<String, InternetDomainName> parsedDomains = parsedDomainsBuilder.build();
    flowCustomLogic.afterValidation(
        DomainCheckFlowCustomLogic.AfterValidationParameters.newBuilder()
-            .setDomainNames(domainNames)
+            .setDomainNames(parsedDomains)
            // TODO: Use as of date from fee extension v0.12 instead of now, if specified.
            .setAsOfDate(now)
            .build());
-    Set<String> existingIds = checkResourcesExist(DomainBase.class, targetIds, now);
+    ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains =
+        ForeignKeyIndex.load(DomainBase.class, domainNames, now);
    Optional<AllocationTokenExtension> allocationTokenExtension =
        eppInput.getSingleExtension(AllocationTokenExtension.class);
    Optional<AllocationTokenDomainCheckResults> tokenDomainCheckResults =
        allocationTokenExtension.map(
            tokenExtension ->
                allocationTokenFlowUtils.checkDomainsWithToken(
-                    ImmutableList.copyOf(domainNames.values()),
+                    ImmutableList.copyOf(parsedDomains.values()),
                    tokenExtension.getAllocationToken(),
                    clientId,
                    now));
@@ -173,18 +185,18 @@ public final class DomainCheckFlow implements Flow {
            .orElse(ImmutableMap.of());
    Optional<AllocationToken> allocationToken =
        tokenDomainCheckResults.flatMap(AllocationTokenDomainCheckResults::token);
-    for (String targetId : targetIds) {
+    for (String domainName : domainNames) {
      Optional<String> message =
          getMessageForCheck(
-              domainNames.get(targetId),
-              existingIds,
+              parsedDomains.get(domainName),
+              existingDomains,
              domainCheckResults,
              tldStates,
              allocationToken);
      boolean isAvailable = !message.isPresent();
-      checksBuilder.add(DomainCheck.create(isAvailable, targetId, message.orElse(null)));
+      checksBuilder.add(DomainCheck.create(isAvailable, domainName, message.orElse(null)));
      if (isAvailable) {
-        availableDomains.add(targetId);
+        availableDomains.add(domainName);
      }
    }
    BeforeResponseReturnData responseData =
@@ -193,7 +205,11 @@ public final class DomainCheckFlow implements Flow {
                .setDomainChecks(checksBuilder.build())
                .setResponseExtensions(
                    getResponseExtensions(
-                        domainNames, availableDomains.build(), now, allocationToken))
+                        parsedDomains,
+                        existingDomains,
+                        availableDomains.build(),
+                        now,
+                        allocationToken))
                .setAsOfDate(now)
                .build());
    return responseBuilder
@@ -204,11 +220,11 @@ public final class DomainCheckFlow implements Flow {

  private Optional<String> getMessageForCheck(
      InternetDomainName domainName,
-      Set<String> existingIds,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains,
      ImmutableMap<InternetDomainName, String> tokenCheckResults,
-      Map<String, TldState> tldStates,
+      ImmutableMap<String, TldState> tldStates,
      Optional<AllocationToken> allocationToken) {
-    if (existingIds.contains(domainName.toString())) {
+    if (existingDomains.containsKey(domainName.toString())) {
      return Optional.of("In use");
    }
    TldState tldState = tldStates.get(domainName.parent().toString());
@@ -228,6 +244,7 @@ public final class DomainCheckFlow implements Flow {
  /** Handle the fee check extension. */
  private ImmutableList<? extends ResponseExtension> getResponseExtensions(
      ImmutableMap<String, InternetDomainName> domainNames,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains,
      ImmutableSet<String> availableDomains,
      DateTime now,
      Optional<AllocationToken> allocationToken)
@@ -240,6 +257,9 @@ public final class DomainCheckFlow implements Flow {
    FeeCheckCommandExtension<?, ?> feeCheck = feeCheckOpt.get();
    ImmutableList.Builder<FeeCheckResponseExtensionItem> responseItems =
        new ImmutableList.Builder<>();
+    ImmutableMap<String, EppResource> domainObjs =
+        loadDomainsForRestoreChecks(feeCheck, domainNames, existingDomains);
+
    for (FeeCheckCommandExtensionItem feeCheckItem : feeCheck.getItems()) {
      for (String domainName : getDomainNamesToCheckForFee(feeCheckItem, domainNames.keySet())) {
        FeeCheckResponseExtensionItem.Builder<?> builder = feeCheckItem.createResponseBuilder();
@@ -247,7 +267,7 @@ public final class DomainCheckFlow implements Flow {
            feeCheckItem,
            builder,
            domainNames.get(domainName),
-            Optional.empty(),
+            Optional.ofNullable((DomainBase) domainObjs.get(domainName)),
            feeCheck.getCurrency(),
            now,
            pricingLogic,
@@ -259,12 +279,57 @@ public final class DomainCheckFlow implements Flow {
    return ImmutableList.of(feeCheck.createResponse(responseItems.build()));
  }

+  /**
+   * Loads and returns all existing domains that are having restore fees checked.
+   *
+   * <p>This is necessary so that we can check their expiration dates to determine if a one-year
+   * renewal is part of the cost of a restore.
+   *
+   * <p>This may be resource-intensive for large checks of many restore fees, but those are
+   * comparatively rare, and we are at least using an in-memory cache. Also this will get a lot
+   * nicer in Cloud SQL when we can SELECT just the fields we want rather than having to load the
+   * entire entity.
+   */
+  private ImmutableMap<String, EppResource> loadDomainsForRestoreChecks(
+      FeeCheckCommandExtension<?, ?> feeCheck,
+      ImmutableMap<String, InternetDomainName> domainNames,
+      ImmutableMap<String, ForeignKeyIndex<DomainBase>> existingDomains) {
+    ImmutableList<String> restoreCheckDomains;
+    if (feeCheck instanceof FeeCheckCommandExtensionV06) {
+      // The V06 fee extension supports specifying the command fees to check on a per-domain basis.
+      restoreCheckDomains =
+          feeCheck.getItems().stream()
+              .filter(fc -> fc.getCommandName() == CommandName.RESTORE)
+              .map(FeeCheckCommandExtensionItem::getDomainName)
+              .collect(toImmutableList());
+    } else if (feeCheck.getItems().stream()
+        .anyMatch(fc -> fc.getCommandName() == CommandName.RESTORE)) {
+      // The more recent fee extension versions support specifying the command fees to check only on
+      // the overall domain check, not per-domain.
+      restoreCheckDomains = ImmutableList.copyOf(domainNames.keySet());
+    } else {
+      // Fall-through case for more recent fee extension versions when the restore fee isn't being
+      // checked.
+      restoreCheckDomains = ImmutableList.of();
+    }
+
+    // Filter down to just domains we know exist and then use the EppResource cache to load them.
+    ImmutableMap<String, VKey<DomainBase>> existingDomainsToLoad =
+        restoreCheckDomains.stream()
+            .filter(existingDomains::containsKey)
+            .collect(toImmutableMap(d -> d, d -> existingDomains.get(d).getResourceKey()));
+    ImmutableMap<VKey<? extends EppResource>, EppResource> loadedDomains =
+        EppResource.loadCached(ImmutableList.copyOf(existingDomainsToLoad.values()));
+    return ImmutableMap.copyOf(
+        Maps.transformEntries(existingDomainsToLoad, (k, v) -> loadedDomains.get(v)));
+  }
+
  /**
   * Return the domains to be checked for a particular fee check item. Some versions of the fee
   * extension specify the domain name in the extension item, while others use the list of domain
   * names from the regular check domain availability list.
   */
-  private Set<String> getDomainNamesToCheckForFee(
+  private ImmutableSet<String> getDomainNamesToCheckForFee(
      FeeCheckCommandExtensionItem feeCheckItem, ImmutableSet<String> availabilityCheckDomains)
      throws OnlyCheckedNamesCanBeFeeCheckedException {
    if (feeCheckItem.isDomainNameSupported()) {
@@ -48,7 +48,6 @@ import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.model.tmch.ClaimsListShard;
 import google.registry.util.Clock;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import javax.inject.Inject;
@@ -87,14 +86,14 @@ public final class DomainClaimsCheckFlow implements Flow {
    if (eppInput.getSingleExtension(AllocationTokenExtension.class).isPresent()) {
      throw new DomainClaimsCheckNotAllowedWithAllocationTokens();
    }
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
+    ImmutableList<String> domainNames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(domainNames, maxChecks);
    Set<String> seenTlds = new HashSet<>();
    ImmutableList.Builder<LaunchCheck> launchChecksBuilder = new ImmutableList.Builder<>();
-    for (String targetId : ImmutableSet.copyOf(targetIds)) {
-      InternetDomainName domainName = validateDomainName(targetId);
-      validateDomainNameWithIdnTables(domainName);
-      String tld = domainName.parent().toString();
+    for (String domainName : ImmutableSet.copyOf(domainNames)) {
+      InternetDomainName parsedDomain = validateDomainName(domainName);
+      validateDomainNameWithIdnTables(parsedDomain);
+      String tld = parsedDomain.parent().toString();
      // Only validate access to a TLD the first time it is encountered.
      if (seenTlds.add(tld)) {
        if (!isSuperuser) {
@@ -105,10 +104,10 @@ public final class DomainClaimsCheckFlow implements Flow {
          verifyClaimsPeriodNotEnded(registry, now);
        }
      }
-      Optional<String> claimKey = ClaimsListShard.get().getClaimKey(domainName.parts().get(0));
+      Optional<String> claimKey = ClaimsListShard.get().getClaimKey(parsedDomain.parts().get(0));
      launchChecksBuilder.add(
          LaunchCheck.create(
-              LaunchCheckName.create(claimKey.isPresent(), targetId), claimKey.orElse(null)));
+              LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null)));
    }
    return responseBuilder
        .setOnlyExtension(LaunchCheckResponseExtension.create(CLAIMS, launchChecksBuilder.build()))
@@ -17,6 +17,7 @@ package google.registry.flows.domain;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Predicates.equalTo;
+import static com.google.common.base.Strings.emptyToNull;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
@@ -32,7 +33,9 @@ import static google.registry.model.registry.Registry.TldState.GENERAL_AVAILABIL
 import static google.registry.model.registry.Registry.TldState.PREDELEGATION;
 import static google.registry.model.registry.Registry.TldState.QUIET_PERIOD;
 import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
+import static google.registry.model.registry.label.ReservationType.ALLOWED_IN_SUNRISE;
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
+import static google.registry.model.registry.label.ReservationType.NAME_COLLISION;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_ANCHOR_TENANT;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_SPECIFIC_USE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -87,6 +90,7 @@ import google.registry.model.domain.DomainCommand.InvalidReferencesException;
 import google.registry.model.domain.DomainCommand.Update;
 import google.registry.model.domain.ForeignKeyedDesignatedContact;
 import google.registry.model.domain.Period;
+import google.registry.model.domain.fee.BaseFee;
 import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Credit;
 import google.registry.model.domain.fee.Fee;
@@ -152,9 +156,7 @@ public class DomainFlowUtils {

  /** Reservation types that are only allowed in sunrise by policy. */
  public static final ImmutableSet<ReservationType> TYPES_ALLOWED_FOR_CREATE_ONLY_IN_SUNRISE =
-      Sets.immutableEnumSet(
-          ReservationType.ALLOWED_IN_SUNRISE,
-          ReservationType.NAME_COLLISION);
+      Sets.immutableEnumSet(ALLOWED_IN_SUNRISE, NAME_COLLISION);

  /** Warning message for allocation of collision domains in sunrise. */
  public static final String COLLISION_MESSAGE =
@@ -583,15 +585,15 @@ public class DomainFlowUtils {
    builder
        .setCommand(feeRequest.getCommandName(), feeRequest.getPhase(), feeRequest.getSubphase())
        .setCurrencyIfSupported(registry.getCurrency())
-        .setPeriod(feeRequest.getPeriod())
-        .setClass(pricingLogic.getFeeClass(domainNameString, now).orElse(null));
+        .setPeriod(feeRequest.getPeriod());

+    String feeClass = null;
    ImmutableList<Fee> fees = ImmutableList.of();
    switch (feeRequest.getCommandName()) {
      case CREATE:
        // Don't return a create price for reserved names.
        if (isReserved(domainName, isSunrise) && !isAvailable) {
-          builder.setClass("reserved"); // Override whatever class we've set above.
+          feeClass = "reserved";
          builder.setAvailIfSupported(false);
          builder.setReasonIfSupported("reserved");
        } else {
@@ -616,15 +618,11 @@ public class DomainFlowUtils {
          throw new RestoresAreAlwaysForOneYearException();
        }
        builder.setAvailIfSupported(true);
-        // The domain object is present only on domain info commands, not on domain check commands,
-        // because check commands can query up to 50 domains and it isn't performant to load them
-        // all. So, only on info commands can we actually determine if we should include the renewal
-        // fee because the domain needs to have been loaded in order to know its expiration time. We
-        // default to including the renewal fee on domain checks because typically most domains are
-        // deleted during the autorenew grace period and thus if restored will require a renewal,
-        // but this is just a best guess.
+        // Domains that never existed, or that used to exist but have completed the entire deletion
+        // process, don't count as expired for the purposes of requiring an added year of renewal on
+        // restore because they can't be restored in the first place.
        boolean isExpired =
-            !domain.isPresent() || domain.get().getRegistrationExpirationTime().isBefore(now);
+            domain.isPresent() && domain.get().getRegistrationExpirationTime().isBefore(now);
        fees = pricingLogic.getRestorePrice(registry, domainNameString, now, isExpired).getFees();
        break;
      case TRANSFER:
@@ -642,6 +640,23 @@ public class DomainFlowUtils {
        throw new UnknownFeeCommandException(feeRequest.getUnparsedCommandName());
    }

+    if (feeClass == null) {
+      // Calculate and set the correct fee class based on whether the name is a collision name or we
+      // are returning any premium fees, but only if the fee class isn't already set (i.e. because
+      // the domain is reserved, which overrides any other classes).
+      boolean isNameCollisionInSunrise =
+          registry.getTldState(now).equals(START_DATE_SUNRISE)
+              && getReservationTypes(domainName).contains(NAME_COLLISION);
+      boolean isPremium = fees.stream().anyMatch(BaseFee::isPremium);
+      feeClass =
+          emptyToNull(
+              Joiner.on('-')
+                  .skipNulls()
+                  .join(
+                      isPremium ? "premium" : null, isNameCollisionInSunrise ? "collision" : null));
+    }
+    builder.setClass(feeClass);
+
    // Set the fees, and based on the validDateRange of the fees, set the notAfterDate.
    if (!fees.isEmpty()) {
      builder.setFees(fees);
@@ -15,7 +15,6 @@
 package google.registry.flows.domain;

 import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
-import static google.registry.pricing.PricingEngineProxy.getDomainFeeClass;
 import static google.registry.pricing.PricingEngineProxy.getPricesForDomainName;

 import com.google.common.net.InternetDomainName;
@@ -186,11 +185,6 @@ public final class DomainPricingLogic {
            .build());
  }

-  /** Returns the fee class for a given domain and date. */
-  public Optional<String> getFeeClass(String domainName, DateTime dateTime) {
-    return getDomainFeeClass(domainName, dateTime);
-  }
-
  /** Returns the domain create cost with allocation-token-related discounts applied. */
  private Money getDomainCreateCostWithDiscount(
      DomainPrices domainPrices, int years, Optional<AllocationToken> allocationToken)
@@ -16,12 +16,12 @@ package google.registry.flows.domain.token;

 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import com.google.common.net.InternetDomainName;
 import google.registry.flows.EppException;
 import google.registry.model.domain.DomainCommand;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.registry.Registry;
-import java.util.function.Function;
 import org.joda.time.DateTime;

 /**
@@ -50,7 +50,6 @@ public class AllocationTokenCustomLogic {
      String clientId,
      DateTime now) {
    // Do nothing.
-    return domainNames.stream()
-        .collect(ImmutableMap.toImmutableMap(Function.identity(), ignored -> ""));
+    return Maps.toMap(domainNames, k -> "");
  }
 }
@@ -19,6 +19,7 @@ import static google.registry.flows.ResourceFlowUtils.verifyTargetIdCount;
 import static google.registry.model.EppResourceUtils.checkResourcesExist;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.flows.EppException;
 import google.registry.flows.ExtensionManager;
@@ -33,8 +34,6 @@ import google.registry.model.host.HostCommand.Check;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.util.Clock;
-import java.util.List;
-import java.util.Set;
 import javax.inject.Inject;

 /**
@@ -59,13 +58,14 @@ public final class HostCheckFlow implements Flow {
  public final EppResponse run() throws EppException {
    extensionManager.validate();  // There are no legal extensions for this flow.
    validateClientIsLoggedIn(clientId);
-    List<String> targetIds = ((Check) resourceCommand).getTargetIds();
-    verifyTargetIdCount(targetIds, maxChecks);
-    Set<String> existingIds = checkResourcesExist(HostResource.class, targetIds, clock.nowUtc());
+    ImmutableList<String> hostnames = ((Check) resourceCommand).getTargetIds();
+    verifyTargetIdCount(hostnames, maxChecks);
+    ImmutableSet<String> existingIds =
+        checkResourcesExist(HostResource.class, hostnames, clock.nowUtc());
    ImmutableList.Builder<HostCheck> checks = new ImmutableList.Builder<>();
-    for (String id : targetIds) {
-      boolean unused = !existingIds.contains(id);
-      checks.add(HostCheck.create(unused, id, unused ? null : "In use"));
+    for (String hostname : hostnames) {
+      boolean unused = !existingIds.contains(hostname);
+      checks.add(HostCheck.create(unused, hostname, unused ? null : "In use"));
    }
    return responseBuilder.setResData(HostCheckData.create(checks.build())).build();
  }
@@ -14,8 +14,8 @@

 package google.registry.flows.poll;

-import static com.google.common.base.Preconditions.checkState;
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
+import static google.registry.flows.poll.PollFlowUtils.ackPollMessage;
 import static google.registry.flows.poll.PollFlowUtils.getPollMessagesQuery;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_NO_MESSAGES;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -100,27 +100,8 @@ public class PollAckFlow implements TransactionalFlow {
    // This keeps track of whether we should include the current acked message in the updated
    // message count that's returned to the user. The only case where we do so is if an autorenew
    // poll message is acked, but its next event is already ready to be delivered.
-    boolean includeAckedMessageInCount = false;
-    if (pollMessage instanceof PollMessage.OneTime) {
-      // One-time poll messages are deleted once acked.
-      ofy().delete().entity(pollMessage);
-    } else {
-      checkState(pollMessage instanceof PollMessage.Autorenew, "Unknown poll message type");
-      PollMessage.Autorenew autorenewPollMessage = (PollMessage.Autorenew) pollMessage;
+    boolean includeAckedMessageInCount = ackPollMessage(pollMessage);

-      // Move the eventTime of this autorenew poll message forward by a year.
-      DateTime nextEventTime = autorenewPollMessage.getEventTime().plusYears(1);
-
-      // If the next event falls within the bounds of the end time, then just update the eventTime
-      // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
-      // autorenew poll message has no more events to deliver and should be deleted.
-      if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
-        ofy().save().entity(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
-        includeAckedMessageInCount = isBeforeOrAt(nextEventTime, now);
-      } else {
-        ofy().delete().entity(autorenewPollMessage);
-      }
-    }
    // We need to return the new queue length. If this was the last message in the queue being
    // acked, then we return a special status code indicating that. Note that the query will
    // include the message being acked.
@@ -14,7 +14,10 @@

 package google.registry.flows.poll;

+import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;

 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.poll.PollMessage;
@@ -33,4 +36,42 @@ public final class PollFlowUtils {
        .filter("eventTime <=", now.toDate())
        .order("eventTime");
  }
+
+  /**
+   * Acknowledges the given {@link PollMessage} and returns whether we should include the current
+   * acked message in the updated message count that's returned to the user.
+   *
+   * <p>The only case where we do so is if an autorenew poll message is acked, but its next event is
+   * already ready to be delivered.
+   */
+  public static boolean ackPollMessage(PollMessage pollMessage) {
+    checkArgument(
+        isBeforeOrAt(pollMessage.getEventTime(), tm().getTransactionTime()),
+        "Cannot ACK poll message with ID %s because its event time is in the future: %s",
+        pollMessage.getId(),
+        pollMessage.getEventTime());
+    boolean includeAckedMessageInCount = false;
+    if (pollMessage instanceof PollMessage.OneTime) {
+      // One-time poll messages are deleted once acked.
+      tm().delete(pollMessage.createVKey());
+    } else if (pollMessage instanceof PollMessage.Autorenew) {
+      PollMessage.Autorenew autorenewPollMessage = (PollMessage.Autorenew) pollMessage;
+
+      // Move the eventTime of this autorenew poll message forward by a year.
+      DateTime nextEventTime = autorenewPollMessage.getEventTime().plusYears(1);
+
+      // If the next event falls within the bounds of the end time, then just update the eventTime
+      // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
+      // autorenew poll message has no more events to deliver and should be deleted.
+      if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
+        tm().saveNewOrUpdate(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
+        includeAckedMessageInCount = isBeforeOrAt(nextEventTime, tm().getTransactionTime());
+      } else {
+        tm().delete(autorenewPollMessage.createVKey());
+      }
+    } else {
+      throw new IllegalArgumentException("Unknown poll message type: " + pollMessage.getClass());
+    }
+    return includeAckedMessageInCount;
+  }
 }
@@ -17,7 +17,7 @@ package google.registry.keyring.kms;
 import google.registry.keyring.api.KeyringException;

 /** An abstraction to simplify Cloud KMS operations. */
-interface KmsConnection {
+public interface KmsConnection {

  /**
   * The maximum allowable secret size, as set by Cloud KMS.
@@ -25,11 +25,9 @@ import com.google.api.services.cloudkms.v1.model.DecryptRequest;
 import com.google.api.services.cloudkms.v1.model.EncryptRequest;
 import com.google.api.services.cloudkms.v1.model.KeyRing;
 import com.google.api.services.cloudkms.v1.model.UpdateCryptoKeyPrimaryVersionRequest;
-import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyringException;
 import google.registry.util.Retrier;
 import java.io.IOException;
-import javax.inject.Inject;

 /** The {@link KmsConnection} which talks to Cloud KMS. */
 class KmsConnectionImpl implements KmsConnection {
@@ -44,12 +42,7 @@ class KmsConnectionImpl implements KmsConnection {
  private final String projectId;
  private final Retrier retrier;

-  @Inject
-  KmsConnectionImpl(
-      @Config("cloudKmsProjectId") String projectId,
-      @Config("cloudKmsKeyRing") String kmsKeyRingName,
-      Retrier retrier,
-      CloudKMS kms) {
+  KmsConnectionImpl(String projectId, String kmsKeyRingName, Retrier retrier, CloudKMS kms) {
    this.projectId = projectId;
    this.kmsKeyRingName = kmsKeyRingName;
    this.retrier = retrier;
@@ -21,6 +21,7 @@ import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;

 import com.googlecode.objectify.Key;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeySerializer;
 import google.registry.keyring.api.Keyring;
 import google.registry.keyring.api.KeyringException;
@@ -86,7 +87,7 @@ public class KmsKeyring implements Keyring {
  private final KmsConnection kmsConnection;

  @Inject
-  KmsKeyring(KmsConnection kmsConnection) {
+  KmsKeyring(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
    this.kmsConnection = kmsConnection;
  }

@@ -24,6 +24,7 @@ import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.Keyring;
 import google.registry.util.GoogleCredentialsBundle;
+import google.registry.util.Retrier;

 /** Dagger module for Cloud KMS. */
@Module
@@ -32,9 +33,22 @@ public abstract class KmsModule {
  public static final String NAME = "KMS";

  @Provides
+  @Config("defaultKms")
  static CloudKMS provideKms(
      @DefaultCredential GoogleCredentialsBundle credentialsBundle,
      @Config("cloudKmsProjectId") String projectId) {
+    return createKms(credentialsBundle, projectId);
+  }
+
+  @Provides
+  @Config("beamKms")
+  static CloudKMS provideBeamKms(
+      @DefaultCredential GoogleCredentialsBundle credentialsBundle,
+      @Config("beamCloudKmsProjectId") String projectId) {
+    return createKms(credentialsBundle, projectId);
+  }
+
+  private static CloudKMS createKms(GoogleCredentialsBundle credentialsBundle, String projectId) {
    return new CloudKMS.Builder(
            credentialsBundle.getHttpTransport(),
            credentialsBundle.getJsonFactory(),
@@ -43,11 +57,28 @@ public abstract class KmsModule {
        .build();
  }

+  @Provides
+  @Config("defaultKmsConnection")
+  static KmsConnection provideKmsConnection(
+      @Config("cloudKmsProjectId") String projectId,
+      @Config("cloudKmsKeyRing") String keyringName,
+      Retrier retrier,
+      @Config("defaultKms") CloudKMS defaultKms) {
+    return new KmsConnectionImpl(projectId, keyringName, retrier, defaultKms);
+  }
+
+  @Provides
+  @Config("beamKmsConnection")
+  static KmsConnection provideBeamKmsConnection(
+      @Config("beamCloudKmsProjectId") String projectId,
+      @Config("beamCloudKmsKeyRing") String keyringName,
+      Retrier retrier,
+      @Config("beamKms") CloudKMS defaultKms) {
+    return new KmsConnectionImpl(projectId, keyringName, retrier, defaultKms);
+  }
+
  @Binds
  @IntoMap
  @StringKey(NAME)
  abstract Keyring provideKeyring(KmsKeyring keyring);
-
-  @Binds
-  abstract KmsConnection provideKmsConnection(KmsConnectionImpl kmsConnectionImpl);
 }
@@ -39,6 +39,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

 import com.google.common.collect.ImmutableMap;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeySerializer;
 import google.registry.keyring.kms.KmsKeyring.PrivateKeyLabel;
 import google.registry.keyring.kms.KmsKeyring.PublicKeyLabel;
@@ -64,7 +65,7 @@ public final class KmsUpdater {
  private final HashMap<String, byte[]> secretValues;

  @Inject
-  public KmsUpdater(KmsConnection kmsConnection) {
+  public KmsUpdater(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
    this.kmsConnection = kmsConnection;

    // Use LinkedHashMap to preserve insertion order on update() to simplify testing and debugging
@@ -16,7 +16,6 @@ package google.registry.model;

 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
@@ -47,7 +46,6 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
-import java.util.stream.StreamSupport;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
@@ -357,16 +355,18 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
        @Override
        public Map<VKey<? extends EppResource>, EppResource> loadAll(
            Iterable<? extends VKey<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> loadAsMap(keys));
+          return tm().doTransactionless(() -> tm().load(keys));
        }
      };

  /**
   * A limited size, limited time cache for EPP resource entities.
   *
-   * <p>This is only used to cache contacts and hosts for the purposes of checking whether they are
-   * deleted or in pending delete during a few domain flows. Any operations on contacts and hosts
-   * directly should of course never use the cache.
+   * <p>This is used to cache contacts and hosts for the purposes of checking whether they are
+   * deleted or in pending delete during a few domain flows, and also to cache domains for the
+   * purpose of determining restore fees in domain checks. Any mutating operations directly on EPP
+   * resources should of course never use the cache as they always need perfectly up-to-date
+   * information.
   */
  @NonFinalForTesting
  private static LoadingCache<VKey<? extends EppResource>, EppResource> cacheEppResources =
@@ -395,7 +395,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
  public static ImmutableMap<VKey<? extends EppResource>, EppResource> loadCached(
      Iterable<VKey<? extends EppResource>> keys) {
    if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return loadAsMap(keys);
+      return tm().load(keys);
    }
    try {
      return cacheEppResources.getAll(keys);
@@ -423,17 +423,4 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
      throw new RuntimeException("Error loading cached EppResources", e.getCause());
    }
  }
-
-  private static ImmutableMap<VKey<? extends EppResource>, EppResource> loadAsMap(
-      Iterable<? extends VKey<? extends EppResource>> keys) {
-    return StreamSupport.stream(keys.spliterator(), false)
-        // It's possible for us to receive the same key more than once which causes
-        // the immutable map build to break with a duplicate key, so we have to ensure key
-        // uniqueness.
-        .distinct()
-        // We have to use "key -> key" here instead of the identity() function, because
-        // the latter breaks the fairly complicated generic type checking required by the
-        // caching interface.
-        .collect(toImmutableMap(key -> key, key -> tm().load(key)));
-  }
 }
@@ -23,6 +23,7 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.Result;
@@ -45,7 +46,6 @@ import google.registry.model.transfer.TransferStatus;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
 import javax.annotation.Nullable;
 import org.joda.time.DateTime;
@@ -169,7 +169,7 @@ public final class EppResourceUtils {
   * @param uniqueIds a list of ids to match
   * @param now the logical time of the check
   */
-  public static <T extends EppResource> Set<String> checkResourcesExist(
+  public static <T extends EppResource> ImmutableSet<String> checkResourcesExist(
      Class<T> clazz, List<String> uniqueIds, final DateTime now) {
    return ForeignKeyIndex.load(clazz, uniqueIds, now).keySet();
  }
@@ -347,7 +347,27 @@ public abstract class BillingEvent extends ImmutableObject

    @Override
    public VKey<OneTime> createVKey() {
-      return VKey.createOfy(getClass(), Key.create(this));
+      return VKey.create(
+          getClass(),
+          parent.getParent().getName() + "/" + parent.getId() + "/" + getId(),
+          Key.create(this));
+    }
+
+    public static VKey<OneTime> createVKey(Key<OneTime> key) {
+      // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary
+      // key type for the table, which is a single long integer.
+      if (key == null) {
+        return null;
+      }
+      Key parent = key.getParent();
+      Key grandparent = (parent != null) ? parent.getParent() : null;
+      String path =
+          (grandparent != null ? grandparent.getName() : "")
+              + "/"
+              + (parent != null ? parent.getId() : "")
+              + "/"
+              + key.getId();
+      return VKey.create(OneTime.class, path, key);
    }

    @Override
@@ -485,7 +505,21 @@ public abstract class BillingEvent extends ImmutableObject

    @Override
    public VKey<Recurring> createVKey() {
-      return VKey.createOfy(getClass(), Key.create(this));
+      return VKey.create(
+          getClass(),
+          parent.getParent().getName() + "/" + parent.getId() + "/" + getId(),
+          Key.create(this));
+    }
+
+    public static VKey<Recurring> createVKey(Key<Recurring> key) {
+      // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary
+      // key type for the table, which is a single long integer.
+      if (key == null) {
+        return null;
+      }
+      String path =
+          key.getParent().getParent().getName() + "/" + key.getParent().getId() + "/" + key.getId();
+      return VKey.create(Recurring.class, path, key);
    }

    @Override
@@ -596,18 +630,30 @@ public abstract class BillingEvent extends ImmutableObject
          .setParent(historyEntry);
      // Set the grace period's billing event using the appropriate Cancellation builder method.
      if (gracePeriod.getOneTimeBillingEvent() != null) {
-        builder.setOneTimeEventKey(
-            VKey.createOfy(BillingEvent.OneTime.class, gracePeriod.getOneTimeBillingEvent()));
+        builder.setOneTimeEventKey(VKey.from(gracePeriod.getOneTimeBillingEvent()));
      } else if (gracePeriod.getRecurringBillingEvent() != null) {
-        builder.setRecurringEventKey(
-            VKey.createOfy(BillingEvent.Recurring.class, gracePeriod.getRecurringBillingEvent()));
+        builder.setRecurringEventKey(VKey.from(gracePeriod.getRecurringBillingEvent()));
      }
      return builder.build();
    }

    @Override
    public VKey<Cancellation> createVKey() {
-      return VKey.createOfy(getClass(), Key.create(this));
+      return VKey.create(
+          getClass(),
+          parent.getParent().getName() + "/" + parent.getId() + "/" + getId(),
+          Key.create(this));
+    }
+
+    public static VKey<Cancellation> createVKey(Key<Cancellation> key) {
+      // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary
+      // key type for the table, which is a single long integer.
+      if (key == null) {
+        return null;
+      }
+      String path =
+          key.getParent().getParent().getName() + "/" + key.getParent().getId() + "/" + key.getId();
+      return VKey.create(Cancellation.class, path, key);
    }

    @Override
@@ -687,7 +733,21 @@ public abstract class BillingEvent extends ImmutableObject

    @Override
    public VKey<Modification> createVKey() {
-      return VKey.createOfy(this.getClass(), Key.create(this));
+      return VKey.create(
+          getClass(),
+          parent.getParent().getName() + "/" + parent.getId() + "/" + getId(),
+          Key.create(this));
+    }
+
+    public static VKey<Modification> createVKey(Key<Modification> key) {
+      // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary
+      // key type for the table, which is a single long integer.
+      if (key == null) {
+        return null;
+      }
+      String path =
+          key.getParent().getParent().getName() + "/" + key.getParent().getId() + "/" + key.getId();
+      return VKey.create(Modification.class, path, key);
    }

    /**
@@ -0,0 +1,398 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.contact;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
+
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.IgnoreSave;
+import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.condition.IfNull;
+import google.registry.model.EppResource;
+import google.registry.model.EppResource.ResourceWithTransferData;
+import google.registry.model.transfer.ContactTransferData;
+import google.registry.persistence.VKey;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.Embedded;
+import javax.persistence.MappedSuperclass;
+import javax.xml.bind.annotation.XmlElement;
+import org.joda.time.DateTime;
+
+/**
+ * A persistable contact resource including mutable and non-mutable fields.
+ *
+ * @see <a href="https://tools.ietf.org/html/rfc5733">RFC 5733</a>
+ *     <p>This class deliberately does not include an {@link javax.persistence.Id} so that any
+ *     foreign-keyed fields can refer to the proper parent entity's ID, whether we're storing this
+ *     in the DB itself or as part of another entity
+ */
+@MappedSuperclass
+@Embeddable
+@Access(AccessType.FIELD)
+public class ContactBase extends EppResource implements ResourceWithTransferData {
+
+  /**
+   * Unique identifier for this contact.
+   *
+   * <p>This is only unique in the sense that for any given lifetime specified as the time range
+   * from (creationTime, deletionTime) there can only be one contact in Datastore with this id.
+   * However, there can be many contacts with the same id and non-overlapping lifetimes.
+   */
+  String contactId;
+
+  /**
+   * Localized postal info for the contact. All contained values must be representable in the 7-bit
+   * US-ASCII character set. Personal info; cleared by {@link ContactResource.Builder#wipeOut}.
+   */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_local_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_local_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_local_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_local_country_code"))
+  })
+  PostalInfo localizedPostalInfo;
+
+  /**
+   * Internationalized postal info for the contact. Personal info; cleared by {@link
+   * ContactResource.Builder#wipeOut}.
+   */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
+    @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
+    @AttributeOverride(
+        name = "address.streetLine1",
+        column = @Column(name = "addr_i18n_street_line1")),
+    @AttributeOverride(
+        name = "address.streetLine2",
+        column = @Column(name = "addr_i18n_street_line2")),
+    @AttributeOverride(
+        name = "address.streetLine3",
+        column = @Column(name = "addr_i18n_street_line3")),
+    @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
+    @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
+    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
+    @AttributeOverride(
+        name = "address.countryCode",
+        column = @Column(name = "addr_i18n_country_code"))
+  })
+  PostalInfo internationalizedPostalInfo;
+
+  /**
+   * Contact name used for name searches. This is set automatically to be the internationalized
+   * postal name, or if null, the localized postal name, or if that is null as well, null. Personal
+   * info; cleared by {@link ContactResource.Builder#wipeOut}.
+   */
+  @Index String searchName;
+
+  /** Contact’s voice number. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
+  })
+  ContactPhoneNumber voice;
+
+  /** Contact’s fax number. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
+    @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
+  })
+  ContactPhoneNumber fax;
+
+  /** Contact’s email address. Personal info; cleared by {@link ContactResource.Builder#wipeOut}. */
+  @IgnoreSave(IfNull.class)
+  String email;
+
+  /** Authorization info (aka transfer secret) of the contact. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
+    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
+  })
+  ContactAuthInfo authInfo;
+
+  /** Data about any pending or past transfers on this contact. */
+  ContactTransferData transferData;
+
+  /**
+   * The time that this resource was last transferred.
+   *
+   * <p>Can be null if the resource has never been transferred.
+   */
+  DateTime lastTransferTime;
+
+  // If any new fields are added which contain personal information, make sure they are cleared by
+  // the wipeOut() function, so that data is not kept around for deleted contacts.
+
+  /** Disclosure policy. */
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
+    @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
+    @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
+    @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
+    @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
+    @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
+    @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
+  })
+  Disclose disclose;
+
+  @Override
+  public VKey<? extends ContactBase> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.create(ContactBase.class, getRepoId(), Key.create(this));
+  }
+
+  public String getContactId() {
+    return contactId;
+  }
+
+  public PostalInfo getLocalizedPostalInfo() {
+    return localizedPostalInfo;
+  }
+
+  public PostalInfo getInternationalizedPostalInfo() {
+    return internationalizedPostalInfo;
+  }
+
+  public String getSearchName() {
+    return searchName;
+  }
+
+  public ContactPhoneNumber getVoiceNumber() {
+    return voice;
+  }
+
+  public ContactPhoneNumber getFaxNumber() {
+    return fax;
+  }
+
+  public String getEmailAddress() {
+    return email;
+  }
+
+  public ContactAuthInfo getAuthInfo() {
+    return authInfo;
+  }
+
+  public Disclose getDisclose() {
+    return disclose;
+  }
+
+  public final String getCurrentSponsorClientId() {
+    return getPersistedCurrentSponsorClientId();
+  }
+
+  @Override
+  public final ContactTransferData getTransferData() {
+    return Optional.ofNullable(transferData).orElse(ContactTransferData.EMPTY);
+  }
+
+  @Override
+  public DateTime getLastTransferTime() {
+    return lastTransferTime;
+  }
+
+  @Override
+  public String getForeignKey() {
+    return contactId;
+  }
+
+  /**
+   * Postal info for the contact.
+   *
+   * <p>The XML marshalling expects the {@link PostalInfo} objects in a list, but we can't actually
+   * persist them to Datastore that way because Objectify can't handle collections of embedded
+   * objects that themselves contain collections, and there's a list of streets inside. This method
+   * transforms the persisted format to the XML format for marshalling.
+   */
+  @XmlElement(name = "postalInfo")
+  public ImmutableList<PostalInfo> getPostalInfosAsList() {
+    return Stream.of(localizedPostalInfo, internationalizedPostalInfo)
+        .filter(Objects::nonNull)
+        .collect(toImmutableList());
+  }
+
+  @Override
+  public ContactBase cloneProjectedAtTime(DateTime now) {
+    return cloneContactProjectedAtTime(this, now);
+  }
+
+  /**
+   * Clones the contact (or subclass). A separate static method so that we can pass in and return a
+   * T without the compiler complaining.
+   */
+  protected static <T extends ContactBase> T cloneContactProjectedAtTime(T contact, DateTime now) {
+    Builder builder = contact.asBuilder();
+    projectResourceOntoBuilderAtTime(contact, builder, now);
+    return (T) builder.build();
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder<>(clone(this));
+  }
+
+  /** A builder for constructing {@link ContactResource}, since it is immutable. */
+  public static class Builder<T extends ContactBase, B extends Builder<T, B>>
+      extends EppResource.Builder<T, B> implements BuilderWithTransferData<ContactTransferData, B> {
+
+    public Builder() {}
+
+    protected Builder(T instance) {
+      super(instance);
+    }
+
+    public B setContactId(String contactId) {
+      getInstance().contactId = contactId;
+      return thisCastToDerived();
+    }
+
+    public B setLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
+      checkArgument(
+          localizedPostalInfo == null
+              || PostalInfo.Type.LOCALIZED.equals(localizedPostalInfo.getType()));
+      getInstance().localizedPostalInfo = localizedPostalInfo;
+      return thisCastToDerived();
+    }
+
+    public B setInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
+      checkArgument(
+          internationalizedPostalInfo == null
+              || PostalInfo.Type.INTERNATIONALIZED.equals(internationalizedPostalInfo.getType()));
+      getInstance().internationalizedPostalInfo = internationalizedPostalInfo;
+      return thisCastToDerived();
+    }
+
+    public B overlayLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
+      return setLocalizedPostalInfo(
+          getInstance().localizedPostalInfo == null
+              ? localizedPostalInfo
+              : getInstance().localizedPostalInfo.overlay(localizedPostalInfo));
+    }
+
+    public B overlayInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
+      return setInternationalizedPostalInfo(
+          getInstance().internationalizedPostalInfo == null
+              ? internationalizedPostalInfo
+              : getInstance().internationalizedPostalInfo.overlay(internationalizedPostalInfo));
+    }
+
+    public B setVoiceNumber(ContactPhoneNumber voiceNumber) {
+      getInstance().voice = voiceNumber;
+      return thisCastToDerived();
+    }
+
+    public B setFaxNumber(ContactPhoneNumber faxNumber) {
+      getInstance().fax = faxNumber;
+      return thisCastToDerived();
+    }
+
+    public B setEmailAddress(String emailAddress) {
+      getInstance().email = emailAddress;
+      return thisCastToDerived();
+    }
+
+    public B setAuthInfo(ContactAuthInfo authInfo) {
+      getInstance().authInfo = authInfo;
+      return thisCastToDerived();
+    }
+
+    public B setDisclose(Disclose disclose) {
+      getInstance().disclose = disclose;
+      return thisCastToDerived();
+    }
+
+    @Override
+    public B setTransferData(ContactTransferData transferData) {
+      getInstance().transferData = transferData;
+      return thisCastToDerived();
+    }
+
+    @Override
+    public B setLastTransferTime(DateTime lastTransferTime) {
+      getInstance().lastTransferTime = lastTransferTime;
+      return thisCastToDerived();
+    }
+
+    /**
+     * Remove all personally identifying information about a contact.
+     *
+     * <p>This should be used when deleting a contact so that the soft-deleted entity doesn't
+     * contain information that the registrant requested to be deleted.
+     */
+    public B wipeOut() {
+      setEmailAddress(null);
+      setFaxNumber(null);
+      setInternationalizedPostalInfo(null);
+      setLocalizedPostalInfo(null);
+      setVoiceNumber(null);
+      return thisCastToDerived();
+    }
+
+    @Override
+    public T build() {
+      T instance = getInstance();
+      // If TransferData is totally empty, set it to null.
+      if (ContactTransferData.EMPTY.equals(instance.transferData)) {
+        setTransferData(null);
+      }
+      // Set the searchName using the internationalized and localized postal info names.
+      if ((instance.internationalizedPostalInfo != null)
+          && (instance.internationalizedPostalInfo.getName() != null)) {
+        instance.searchName = instance.internationalizedPostalInfo.getName();
+      } else if ((instance.localizedPostalInfo != null)
+          && (instance.localizedPostalInfo.getName() != null)) {
+        instance.searchName = instance.localizedPostalInfo.getName();
+      } else {
+        instance.searchName = null;
+      }
+      return super.build();
+    }
+  }
+}
@@ -0,0 +1,88 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.contact;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.EppResource;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+
+/**
+ * A persisted history entry representing an EPP modification to a contact.
+ *
+ * <p>In addition to the general history fields (e.g. action time, registrar ID) we also persist a
+ * copy of the host entity at this point in time. We persist a raw {@link ContactBase} so that the
+ * foreign-keyed fields in that class can refer to this object.
+ */
+@Entity
+@javax.persistence.Table(
+    indexes = {
+      @javax.persistence.Index(columnList = "creationTime"),
+      @javax.persistence.Index(columnList = "historyRegistrarId"),
+      @javax.persistence.Index(columnList = "historyType"),
+      @javax.persistence.Index(columnList = "historyModificationTime")
+    })
+public class ContactHistory extends HistoryEntry {
+  // Store ContactBase instead of ContactResource so we don't pick up its @Id
+  ContactBase contactBase;
+
+  @Column(nullable = false)
+  VKey<ContactResource> contactRepoId;
+
+  /** The state of the {@link ContactBase} object at this point in time. */
+  public ContactBase getContactBase() {
+    return contactBase;
+  }
+
+  /** The key to the {@link ContactResource} this is based off of. */
+  public VKey<ContactResource> getContactRepoId() {
+    return contactRepoId;
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder(clone(this));
+  }
+
+  public static class Builder extends HistoryEntry.Builder<ContactHistory, ContactHistory.Builder> {
+
+    public Builder() {}
+
+    public Builder(ContactHistory instance) {
+      super(instance);
+    }
+
+    public Builder setContactBase(ContactBase contactBase) {
+      getInstance().contactBase = contactBase;
+      return this;
+    }
+
+    public Builder setContactRepoId(VKey<ContactResource> contactRepoId) {
+      getInstance().contactRepoId = contactRepoId;
+      contactRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
+      return this;
+    }
+
+    // We can remove this once all HistoryEntries are converted to History objects
+    @Override
+    public Builder setParent(Key<? extends EppResource> parent) {
+      super.setParent(parent);
+      getInstance().contactRepoId = VKey.create(ContactResource.class, parent.getName(), parent);
+      return this;
+    }
+  }
+}
@@ -14,36 +14,16 @@

 package google.registry.model.contact;

-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
-
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
-import com.googlecode.objectify.annotation.IgnoreSave;
-import com.googlecode.objectify.annotation.Index;
-import com.googlecode.objectify.condition.IfNull;
-import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
-import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
-import google.registry.model.contact.PostalInfo.Type;
-import google.registry.model.transfer.ContactTransferData;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.stream.Stream;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
-import javax.persistence.AttributeOverride;
-import javax.persistence.AttributeOverrides;
-import javax.persistence.Column;
-import javax.persistence.Embedded;
-import javax.xml.bind.annotation.XmlElement;
 import org.joda.time.DateTime;

 /**
@@ -66,142 +46,13 @@ import org.joda.time.DateTime;
@ExternalMessagingName("contact")
@WithStringVKey
@Access(AccessType.FIELD)
-public class ContactResource extends EppResource
-    implements DatastoreAndSqlEntity, ForeignKeyedEppResource, ResourceWithTransferData {
-
-  /**
-   * Unique identifier for this contact.
-   *
-   * <p>This is only unique in the sense that for any given lifetime specified as the time range
-   * from (creationTime, deletionTime) there can only be one contact in Datastore with this id.
-   * However, there can be many contacts with the same id and non-overlapping lifetimes.
-   */
-  String contactId;
-
-  /**
-   * Localized postal info for the contact. All contained values must be representable in the 7-bit
-   * US-ASCII character set. Personal info; cleared by {@link Builder#wipeOut}.
-   */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "addr_local_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "addr_local_org")),
-    @AttributeOverride(name = "type", column = @Column(name = "addr_local_type")),
-    @AttributeOverride(
-        name = "address.streetLine1",
-        column = @Column(name = "addr_local_street_line1")),
-    @AttributeOverride(
-        name = "address.streetLine2",
-        column = @Column(name = "addr_local_street_line2")),
-    @AttributeOverride(
-        name = "address.streetLine3",
-        column = @Column(name = "addr_local_street_line3")),
-    @AttributeOverride(name = "address.city", column = @Column(name = "addr_local_city")),
-    @AttributeOverride(name = "address.state", column = @Column(name = "addr_local_state")),
-    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_local_zip")),
-    @AttributeOverride(
-        name = "address.countryCode",
-        column = @Column(name = "addr_local_country_code"))
-  })
-  PostalInfo localizedPostalInfo;
-
-  /**
-   * Internationalized postal info for the contact. Personal info; cleared by {@link
-   * Builder#wipeOut}.
-   */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "addr_i18n_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "addr_i18n_org")),
-    @AttributeOverride(name = "type", column = @Column(name = "addr_i18n_type")),
-    @AttributeOverride(
-        name = "address.streetLine1",
-        column = @Column(name = "addr_i18n_street_line1")),
-    @AttributeOverride(
-        name = "address.streetLine2",
-        column = @Column(name = "addr_i18n_street_line2")),
-    @AttributeOverride(
-        name = "address.streetLine3",
-        column = @Column(name = "addr_i18n_street_line3")),
-    @AttributeOverride(name = "address.city", column = @Column(name = "addr_i18n_city")),
-    @AttributeOverride(name = "address.state", column = @Column(name = "addr_i18n_state")),
-    @AttributeOverride(name = "address.zip", column = @Column(name = "addr_i18n_zip")),
-    @AttributeOverride(
-        name = "address.countryCode",
-        column = @Column(name = "addr_i18n_country_code"))
-  })
-  PostalInfo internationalizedPostalInfo;
-
-  /**
-   * Contact name used for name searches. This is set automatically to be the internationalized
-   * postal name, or if null, the localized postal name, or if that is null as well, null. Personal
-   * info; cleared by {@link Builder#wipeOut}.
-   */
-  @Index
-  String searchName;
-
-  /** Contact’s voice number. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "phoneNumber", column = @Column(name = "voice_phone_number")),
-    @AttributeOverride(name = "extension", column = @Column(name = "voice_phone_extension")),
-  })
-  ContactPhoneNumber voice;
-
-  /** Contact’s fax number. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "phoneNumber", column = @Column(name = "fax_phone_number")),
-    @AttributeOverride(name = "extension", column = @Column(name = "fax_phone_extension")),
-  })
-  ContactPhoneNumber fax;
-
-  /** Contact’s email address. Personal info; cleared by {@link Builder#wipeOut}. */
-  @IgnoreSave(IfNull.class)
-  String email;
-
-  /** Authorization info (aka transfer secret) of the contact. */
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "pw.value", column = @Column(name = "auth_info_value")),
-    @AttributeOverride(name = "pw.repoId", column = @Column(name = "auth_info_repo_id")),
-  })
-  ContactAuthInfo authInfo;
-
-  /** Data about any pending or past transfers on this contact. */
-  ContactTransferData transferData;
-
-  /**
-   * The time that this resource was last transferred.
-   *
-   * <p>Can be null if the resource has never been transferred.
-   */
-  DateTime lastTransferTime;
-
-  // If any new fields are added which contain personal information, make sure they are cleared by
-  // the wipeOut() function, so that data is not kept around for deleted contacts.
-
-  /** Disclosure policy. */
-  @Embedded
-  @AttributeOverrides({
-    @AttributeOverride(name = "name", column = @Column(name = "disclose_types_name")),
-    @AttributeOverride(name = "org", column = @Column(name = "disclose_types_org")),
-    @AttributeOverride(name = "addr", column = @Column(name = "disclose_types_addr")),
-    @AttributeOverride(name = "flag", column = @Column(name = "disclose_mode_flag")),
-    @AttributeOverride(name = "voice.marked", column = @Column(name = "disclose_show_voice")),
-    @AttributeOverride(name = "fax.marked", column = @Column(name = "disclose_show_fax")),
-    @AttributeOverride(name = "email.marked", column = @Column(name = "disclose_show_email"))
-  })
-  Disclose disclose;
+public class ContactResource extends ContactBase
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource {

  @Override
  public VKey<ContactResource> createVKey() {
    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
-    return VKey.createOfy(ContactResource.class, Key.create(this));
+    return VKey.create(ContactResource.class, getRepoId(), Key.create(this));
  }

  @Override
@@ -211,81 +62,9 @@ public class ContactResource extends EppResource
    return super.getRepoId();
  }

-  public String getContactId() {
-    return contactId;
-  }
-
-  public PostalInfo getLocalizedPostalInfo() {
-    return localizedPostalInfo;
-  }
-
-  public PostalInfo getInternationalizedPostalInfo() {
-    return internationalizedPostalInfo;
-  }
-
-  public String getSearchName() {
-    return searchName;
-  }
-
-  public ContactPhoneNumber getVoiceNumber() {
-    return voice;
-  }
-
-  public ContactPhoneNumber getFaxNumber() {
-    return fax;
-  }
-
-  public String getEmailAddress() {
-    return email;
-  }
-
-  public ContactAuthInfo getAuthInfo() {
-    return authInfo;
-  }
-
-  public Disclose getDisclose() {
-    return disclose;
-  }
-
-  public final String getCurrentSponsorClientId() {
-    return getPersistedCurrentSponsorClientId();
-  }
-
-  @Override
-  public final ContactTransferData getTransferData() {
-    return Optional.ofNullable(transferData).orElse(ContactTransferData.EMPTY);
-  }
-
-  @Override
-  public DateTime getLastTransferTime() {
-    return lastTransferTime;
-  }
-
-  @Override
-  public String getForeignKey() {
-    return contactId;
-  }
-
-  /**
-   * Postal info for the contact.
-   *
-   * <p>The XML marshalling expects the {@link PostalInfo} objects in a list, but we can't actually
-   * persist them to Datastore that way because Objectify can't handle collections of embedded
-   * objects that themselves contain collections, and there's a list of streets inside. This method
-   * transforms the persisted format to the XML format for marshalling.
-   */
-  @XmlElement(name = "postalInfo")
-  public ImmutableList<PostalInfo> getPostalInfosAsList() {
-    return Stream.of(localizedPostalInfo, internationalizedPostalInfo)
-        .filter(Objects::nonNull)
-        .collect(toImmutableList());
-  }
-
  @Override
  public ContactResource cloneProjectedAtTime(DateTime now) {
-    Builder builder = this.asBuilder();
-    projectResourceOntoBuilderAtTime(this, builder, now);
-    return builder.build();
+    return ContactBase.cloneContactProjectedAtTime(this, now);
  }

  @Override
@@ -294,116 +73,12 @@ public class ContactResource extends EppResource
  }

  /** A builder for constructing {@link ContactResource}, since it is immutable. */
-  public static class Builder extends EppResource.Builder<ContactResource, Builder>
-      implements BuilderWithTransferData<ContactTransferData, Builder> {
+  public static class Builder extends ContactBase.Builder<ContactResource, Builder> {

    public Builder() {}

    private Builder(ContactResource instance) {
      super(instance);
    }
-
-    public Builder setContactId(String contactId) {
-      getInstance().contactId = contactId;
-      return this;
-    }
-
-    public Builder setLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
-      checkArgument(localizedPostalInfo == null
-          || Type.LOCALIZED.equals(localizedPostalInfo.getType()));
-      getInstance().localizedPostalInfo = localizedPostalInfo;
-      return this;
-    }
-
-    public Builder setInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
-      checkArgument(internationalizedPostalInfo == null
-          || Type.INTERNATIONALIZED.equals(internationalizedPostalInfo.getType()));
-      getInstance().internationalizedPostalInfo = internationalizedPostalInfo;
-      return this;
-    }
-
-    public Builder overlayLocalizedPostalInfo(PostalInfo localizedPostalInfo) {
-      return setLocalizedPostalInfo(getInstance().localizedPostalInfo == null
-          ? localizedPostalInfo
-          : getInstance().localizedPostalInfo.overlay(localizedPostalInfo));
-    }
-
-    public Builder overlayInternationalizedPostalInfo(PostalInfo internationalizedPostalInfo) {
-      return setInternationalizedPostalInfo(getInstance().internationalizedPostalInfo == null
-          ? internationalizedPostalInfo
-          : getInstance().internationalizedPostalInfo.overlay(internationalizedPostalInfo));
-    }
-
-    public Builder setVoiceNumber(ContactPhoneNumber voiceNumber) {
-      getInstance().voice = voiceNumber;
-      return this;
-    }
-
-    public Builder setFaxNumber(ContactPhoneNumber faxNumber) {
-      getInstance().fax = faxNumber;
-      return this;
-    }
-
-    public Builder setEmailAddress(String emailAddress) {
-      getInstance().email = emailAddress;
-      return this;
-    }
-
-    public Builder setAuthInfo(ContactAuthInfo authInfo) {
-      getInstance().authInfo = authInfo;
-      return this;
-    }
-
-    public Builder setDisclose(Disclose disclose) {
-      getInstance().disclose = disclose;
-      return this;
-    }
-
-    @Override
-    public Builder setTransferData(ContactTransferData transferData) {
-      getInstance().transferData = transferData;
-      return this;
-    }
-
-    @Override
-    public Builder setLastTransferTime(DateTime lastTransferTime) {
-      getInstance().lastTransferTime = lastTransferTime;
-      return thisCastToDerived();
-    }
-
-    /**
-     * Remove all personally identifying information about a contact.
-     *
-     * <p>This should be used when deleting a contact so that the soft-deleted entity doesn't
-     * contain information that the registrant requested to be deleted.
-     */
-    public Builder wipeOut() {
-      setEmailAddress(null);
-      setFaxNumber(null);
-      setInternationalizedPostalInfo(null);
-      setLocalizedPostalInfo(null);
-      setVoiceNumber(null);
-      return this;
-    }
-
-    @Override
-    public ContactResource build() {
-      ContactResource instance = getInstance();
-      // If TransferData is totally empty, set it to null.
-      if (ContactTransferData.EMPTY.equals(instance.transferData)) {
-        setTransferData(null);
-      }
-      // Set the searchName using the internationalized and localized postal info names.
-      if ((instance.internationalizedPostalInfo != null)
-          && (instance.internationalizedPostalInfo.getName() != null)) {
-        instance.searchName = instance.internationalizedPostalInfo.getName();
-      } else if ((instance.localizedPostalInfo != null)
-          && (instance.localizedPostalInfo.getName() != null)) {
-        instance.searchName = instance.localizedPostalInfo.getName();
-      } else {
-        instance.searchName = null;
-      }
-      return super.build();
-    }
  }
 }
@@ -85,6 +85,6 @@ public class DesignatedContact extends ImmutableObject {
  }

  public DesignatedContact reconstitute() {
-    return create(type, VKey.createOfy(ContactResource.class, contact));
+    return create(type, VKey.from(contact));
  }
 }
@@ -643,10 +643,6 @@ public class DomainBase extends EppResource
    return VKey.create(DomainBase.class, getRepoId(), Key.create(this));
  }

-  public static VKey<DomainBase> createVKey(Key key) {
-    return VKey.create(DomainBase.class, key.getName(), key);
-  }
-
  /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
  private static final Predicate<DesignatedContact> IS_REGISTRANT =
      (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);
@@ -41,7 +41,6 @@ import google.registry.model.eppinput.ResourceCommand.SingleResourceCommand;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.persistence.VKey;
-import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -447,7 +446,8 @@ public class DomainCommand {
  private static <T extends EppResource> ImmutableMap<String, VKey<T>> loadByForeignKeysCached(
      final Set<String> foreignKeys, final Class<T> clazz, final DateTime now)
      throws InvalidReferencesException {
-    Map<String, ForeignKeyIndex<T>> fkis = ForeignKeyIndex.loadCached(clazz, foreignKeys, now);
+    ImmutableMap<String, ForeignKeyIndex<T>> fkis =
+        ForeignKeyIndex.loadCached(clazz, foreignKeys, now);
    if (!fkis.keySet().equals(foreignKeys)) {
      throw new InvalidReferencesException(
          clazz, ImmutableSet.copyOf(difference(foreignKeys, fkis.keySet())));
@@ -15,7 +15,6 @@
 package google.registry.model.domain.fee;

 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.eppinput.EppInput.CommandExtension;
 import org.joda.money.CurrencyUnit;

@@ -40,7 +39,7 @@ public interface FeeCheckCommandExtension<
   */
  CurrencyUnit getCurrency();

-  ImmutableSet<C> getItems();
+  ImmutableList<C> getItems();

  R createResponse(ImmutableList<? extends FeeCheckResponseExtensionItem> items);
 }
@@ -17,11 +17,10 @@ package google.registry.model.domain.fee06;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;

 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
 import google.registry.model.domain.fee.FeeCheckResponseExtensionItem;
-import java.util.Set;
+import java.util.List;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import org.joda.money.CurrencyUnit;
@@ -34,7 +33,7 @@ public class FeeCheckCommandExtensionV06 extends ImmutableObject
        FeeCheckResponseExtensionV06> {

  @XmlElement(name = "domain")
-  Set<FeeCheckCommandExtensionItemV06> items;
+  List<FeeCheckCommandExtensionItemV06> items;

  @Override
  public CurrencyUnit getCurrency() {
@@ -42,7 +41,7 @@ public class FeeCheckCommandExtensionV06 extends ImmutableObject
  }

  @Override
-  public ImmutableSet<FeeCheckCommandExtensionItemV06> getItems() {
+  public ImmutableList<FeeCheckCommandExtensionItemV06> getItems() {
    return nullToEmptyImmutableCopy(items);
  }

@@ -17,7 +17,6 @@ package google.registry.model.domain.fee11;
 import static com.google.common.base.Preconditions.checkState;

 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
@@ -69,8 +68,8 @@ public class FeeCheckCommandExtensionV11 extends ImmutableObject
  }

  @Override
-  public ImmutableSet<FeeCheckCommandExtensionItemV11> getItems() {
-    return ImmutableSet.of(new FeeCheckCommandExtensionItemV11());
+  public ImmutableList<FeeCheckCommandExtensionItemV11> getItems() {
+    return ImmutableList.of(new FeeCheckCommandExtensionItemV11());
  }

  @Override
@@ -17,11 +17,10 @@ package google.registry.model.domain.fee12;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;

 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.fee.FeeCheckCommandExtension;
 import google.registry.model.domain.fee.FeeCheckResponseExtensionItem;
-import java.util.Set;
+import java.util.List;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlType;
@@ -43,10 +42,10 @@ public class FeeCheckCommandExtensionV12 extends ImmutableObject
  }

  @XmlElement(name = "command")
-  Set<FeeCheckCommandExtensionItemV12> items;
+  List<FeeCheckCommandExtensionItemV12> items;

  @Override
-  public ImmutableSet<FeeCheckCommandExtensionItemV12> getItems() {
+  public ImmutableList<FeeCheckCommandExtensionItemV12> getItems() {
    return nullToEmptyImmutableCopy(items);
  }

@@ -86,10 +86,7 @@ public enum GracePeriodStatus implements EppEnum {
  /** Provide a quick lookup of GracePeriodStatus from XML name. */
  private static final ImmutableMap<String, GracePeriodStatus> XML_NAME_TO_GRACE_PERIOD_STATUS =
      Stream.of(GracePeriodStatus.values())
-          .collect(
-              toImmutableMap(
-                  (GracePeriodStatus gracePeriodStatus) -> gracePeriodStatus.xmlName,
-                  value -> value));
+          .collect(toImmutableMap(GracePeriodStatus::getXmlName, value -> value));

  @XmlAttribute(name = "s")
  private final String xmlName;
@@ -183,7 +183,7 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
  }

  public VKey<AllocationToken> createVKey() {
-    return VKey.createOfy(getClass(), Key.create(this));
+    return VKey.create(getClass(), getToken(), Key.create(this));
  }

  @Override
@@ -125,8 +125,8 @@ public class HostBase extends EppResource {
  }

  @Override
-  public VKey<? extends EppResource> createVKey() {
-    return VKey.createOfy(HostBase.class, Key.create(this));
+  public VKey<? extends HostBase> createVKey() {
+    return VKey.create(HostBase.class, getRepoId(), Key.create(this));
  }

  @Deprecated
@@ -73,7 +73,7 @@ public class HostHistory extends HistoryEntry {
      return this;
    }

-    public Builder setHostResourceId(VKey<HostResource> hostRepoId) {
+    public Builder setHostRepoId(VKey<HostResource> hostRepoId) {
      getInstance().hostRepoId = hostRepoId;
      hostRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
      return this;
@@ -83,7 +83,7 @@ public class HostHistory extends HistoryEntry {
    @Override
    public Builder setParent(Key<? extends EppResource> parent) {
      super.setParent(parent);
-      getInstance().hostRepoId = VKey.createOfy(HostResource.class, (Key<HostResource>) parent);
+      getInstance().hostRepoId = VKey.create(HostResource.class, parent.getName(), parent);
      return this;
    }
  }
@@ -48,8 +48,7 @@ public class HostResource extends HostBase

  @Override
  public VKey<HostResource> createVKey() {
-    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
-    return VKey.createOfy(HostResource.class, Key.create(this));
+    return VKey.create(HostResource.class, getRepoId(), Key.create(this));
  }

  @Override
@@ -15,7 +15,6 @@
 package google.registry.model.index;

 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static com.google.common.collect.Maps.filterValues;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -178,11 +177,11 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   * <p>The returned map will omit any keys for which the {@link ForeignKeyIndex} doesn't exist or
   * has been soft deleted.
   */
-  public static <E extends EppResource> Map<String, ForeignKeyIndex<E>> load(
+  public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> load(
      Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
-    return filterValues(
-        ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys),
-        (ForeignKeyIndex<?> fki) -> now.isBefore(fki.deletionTime));
+    return ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys).entrySet().stream()
+        .filter(e -> now.isBefore(e.getValue().deletionTime))
+        .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
  }

  static final CacheLoader<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>> CACHE_LOADER =
@@ -249,7 +248,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   * <p>Don't use the cached version of this method unless you really need it for performance
   * reasons, and are OK with the trade-offs in loss of transactional consistency.
   */
-  public static <E extends EppResource> Map<String, ForeignKeyIndex<E>> loadCached(
+  public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> loadCached(
      Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
    if (!RegistryConfig.isEppResourceCachingEnabled()) {
      return tm().doTransactionless(() -> load(clazz, foreignKeys, now));
@@ -262,16 +261,14 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
      // This cast is safe because when we loaded ForeignKeyIndexes above we used type clazz, which
      // is scoped to E.
      @SuppressWarnings("unchecked")
-      Map<String, ForeignKeyIndex<E>> fkisFromCache = cacheForeignKeyIndexes
-          .getAll(fkiKeys)
-          .entrySet()
-          .stream()
-          .filter(entry -> entry.getValue().isPresent())
-          .filter(entry -> now.isBefore(entry.getValue().get().getDeletionTime()))
-          .collect(
-              ImmutableMap.toImmutableMap(
-                  entry -> entry.getKey().getName(),
-                  entry -> (ForeignKeyIndex<E>) entry.getValue().get()));
+      ImmutableMap<String, ForeignKeyIndex<E>> fkisFromCache =
+          cacheForeignKeyIndexes.getAll(fkiKeys).entrySet().stream()
+              .filter(entry -> entry.getValue().isPresent())
+              .filter(entry -> now.isBefore(entry.getValue().get().getDeletionTime()))
+              .collect(
+                  ImmutableMap.toImmutableMap(
+                      entry -> entry.getKey().getName(),
+                      entry -> (ForeignKeyIndex<E>) entry.getValue().get()));
      return fkisFromCache;
    } catch (ExecutionException e) {
      throw new RuntimeException("Error loading cached ForeignKeyIndexes", e.getCause());
@@ -15,15 +15,18 @@
 package google.registry.model.ofy;

 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

+import com.google.common.base.Functions;
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.TransactionManager;
-import java.util.Iterator;
+import java.util.Map.Entry;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
@@ -156,12 +159,15 @@ public class DatastoreTransactionManager implements TransactionManager {
  }

  @Override
-  public <T> ImmutableList<T> load(Iterable<VKey<T>> keys) {
-    Iterator<Key<T>> iter =
-        StreamSupport.stream(keys.spliterator(), false).map(VKey::getOfyKey).iterator();
+  public <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys) {
+    // Keep track of the Key -> VKey mapping so we can translate them back.
+    ImmutableMap<Key<T>, VKey<? extends T>> keyMap =
+        StreamSupport.stream(keys.spliterator(), false)
+            .distinct()
+            .collect(toImmutableMap(key -> (Key<T>) key.getOfyKey(), Functions.identity()));

-    // The lambda argument to keys() effectively converts Iterator -> Iterable.
-    return ImmutableList.copyOf(getOfy().load().keys(() -> iter).values());
+    return getOfy().load().keys(keyMap.keySet()).entrySet().stream()
+        .collect(ImmutableMap.toImmutableMap(entry -> keyMap.get(entry.getKey()), Entry::getValue));
  }

  @Override
@@ -154,6 +154,18 @@ public abstract class PollMessage extends ImmutableObject
  @Override
  public abstract VKey<? extends PollMessage> createVKey();

+  public static VKey<PollMessage> createVKey(Key<PollMessage> key) {
+    // TODO(b/159207551): As it stands, the SQL key generated here doesn't mesh with the primary key
+    // type for the table, which is a single long integer.  Also note that the class id is not
+    // correct here and as such the resulting key will not be loadable from SQL.
+    if (key == null) {
+      return null;
+    }
+    String path =
+        key.getParent().getParent().getName() + "/" + key.getParent().getId() + key.getId();
+    return VKey.create(PollMessage.class, path, key);
+  }
+
  /** Override Buildable.asBuilder() to give this method stronger typing. */
  @Override
  public abstract Builder<?, ?> asBuilder();
@@ -297,7 +309,7 @@ public abstract class PollMessage extends ImmutableObject

    @Override
    public VKey<OneTime> createVKey() {
-      return VKey.createOfy(this.getClass(), Key.create(this));
+      return VKey.create(this.getClass(), getId(), Key.create(this));
    }

    @Override
@@ -398,7 +410,7 @@ public abstract class PollMessage extends ImmutableObject

    @Override
    public VKey<Autorenew> createVKey() {
-      return VKey.createOfy(this.getClass(), Key.create(this));
+      return VKey.create(this.getClass(), getId(), Key.create(this));
    }

    @Override
@@ -14,7 +14,6 @@

 package google.registry.model.pricing;

-import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;

@@ -46,18 +45,12 @@ public interface PremiumPricingEngine {
    // create, renew, restore, and transfer.
    private Money createCost;
    private Money renewCost;
-    private Optional<String> feeClass;

-    static DomainPrices create(
-        boolean isPremium,
-        Money createCost,
-        Money renewCost,
-        Optional<String> feeClass) {
+    static DomainPrices create(boolean isPremium, Money createCost, Money renewCost) {
      DomainPrices instance = new DomainPrices();
      instance.isPremium = isPremium;
      instance.createCost = createCost;
      instance.renewCost = renewCost;
-      instance.feeClass = feeClass;
      return instance;
    }

@@ -75,10 +68,5 @@ public interface PremiumPricingEngine {
    public Money getRenewCost() {
      return renewCost;
    }
-
-    /** Returns the fee class of the cost (used for the Fee extension). */
-    public Optional<String> getFeeClass() {
-      return feeClass;
-    }
  }
 }
@@ -15,14 +15,9 @@
 package google.registry.model.pricing;

 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Strings.emptyToNull;
-import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
 import static google.registry.model.registry.label.PremiumListUtils.getPremiumPrice;
-import static google.registry.model.registry.label.ReservationType.NAME_COLLISION;
-import static google.registry.model.registry.label.ReservedList.getReservationTypes;
 import static google.registry.util.DomainNameUtils.getTldFromDomainName;

-import com.google.common.base.Joiner;
 import com.google.common.net.InternetDomainName;
 import google.registry.model.registry.Registry;
 import java.util.Optional;
@@ -44,16 +39,9 @@ public final class StaticPremiumListPricingEngine implements PremiumPricingEngin
    String label = InternetDomainName.from(fullyQualifiedDomainName).parts().get(0);
    Registry registry = Registry.get(checkNotNull(tld, "tld"));
    Optional<Money> premiumPrice = getPremiumPrice(label, registry);
-    boolean isNameCollisionInSunrise =
-        registry.getTldState(priceTime).equals(START_DATE_SUNRISE)
-            && getReservationTypes(label, tld).contains(NAME_COLLISION);
-    String feeClass = emptyToNull(Joiner.on('-').skipNulls().join(
-            premiumPrice.isPresent() ? "premium" : null,
-            isNameCollisionInSunrise ? "collision" : null));
    return DomainPrices.create(
        premiumPrice.isPresent(),
        premiumPrice.orElse(registry.getStandardCreateCost()),
-        premiumPrice.orElse(registry.getStandardRenewCost(priceTime)),
-        Optional.ofNullable(feeClass));
+        premiumPrice.orElse(registry.getStandardRenewCost(priceTime)));
  }
 }
@@ -19,13 +19,13 @@ import static com.google.common.base.Predicates.equalTo;
 import static com.google.common.base.Predicates.in;
 import static com.google.common.base.Predicates.not;
 import static com.google.common.base.Strings.emptyToNull;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Maps.filterValues;
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

 import com.google.common.base.Joiner;
@@ -37,7 +37,6 @@ import com.google.common.collect.Streams;
 import com.google.common.net.InternetDomainName;
 import com.googlecode.objectify.Key;
 import google.registry.model.registry.Registry.TldType;
-import java.util.Map;
 import java.util.Optional;

 /** Utilities for finding and listing {@link Registry} entities. */
@@ -71,7 +70,7 @@ public final class Registries {
                              .collect(toImmutableSet());
                      return Registry.getAll(tlds).stream()
                          .map(e -> Maps.immutableEntry(e.getTldStr(), e.getTldType()))
-                          .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+                          .collect(entriesToImmutableMap());
                    }));
  }

@@ -339,6 +339,14 @@ public class Registry extends ImmutableObject implements Buildable {
  /** The type of the TLD, whether it's real or for testing. */
  TldType tldType = TldType.REAL;

+  /**
+   * Whether to disable invoicing for a {@link TldType#REAL} TLD.
+   *
+   * <p>Note that invoicing is always disabled for {@link TldType#TEST} TLDs. Setting this field has
+   * no effect for {@link TldType#TEST} TLDs.
+   */
+  boolean disableInvoicing = false;
+
  /**
   * A property that transitions to different TldStates at different times. Stored as a list of
   * TldStateTransition embedded objects using the @Mapify annotation.
@@ -579,8 +587,9 @@ public class Registry extends ImmutableObject implements Buildable {
    return Fee.create(
        eapFeeSchedule.getValueAtTime(now).getAmount(),
        FeeType.EAP,
-        // An EAP fee counts as premium so the domain's overall Fee doesn't show as standard-priced.
-        true,
+        // An EAP fee does not count as premium -- it's a separate one-time fee, independent of
+        // which the domain is separately considered standard vs premium depending on renewal price.
+        false,
        validPeriod,
        validPeriod.upperEndpoint());
  }
@@ -637,6 +646,11 @@ public class Registry extends ImmutableObject implements Buildable {
      return this;
    }

+    public Builder setDisableInvoicing(boolean disableInvoicing) {
+      getInstance().disableInvoicing = disableInvoicing;
+      return this;
+    }
+
    /** Sets the TLD state to transition to the specified states at the specified times. */
    public Builder setTldStateTransitions(ImmutableSortedMap<DateTime, TldState> tldStatesMap) {
      checkNotNull(tldStatesMap, "TLD states map cannot be null");
@@ -34,7 +34,6 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.MapDifference;
-import com.google.common.collect.MapDifference.ValueDifference;
 import com.google.common.collect.Maps;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.util.concurrent.UncheckedExecutionException;
@@ -265,7 +264,7 @@ public final class ReservedList
          datastoreList.reservedListMap.entrySet().parallelStream()
              .collect(
                  toImmutableMap(
-                      entry -> entry.getKey(),
+                      Map.Entry::getKey,
                      entry ->
                          ReservedEntry.create(
                              entry.getValue().reservationType, entry.getValue().comment)));
@@ -283,17 +282,14 @@ public final class ReservedList
                  cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
        } else {
          StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
-          diff.entriesDiffering().entrySet().stream()
+          diff.entriesDiffering()
              .forEach(
-                  entry -> {
-                    String label = entry.getKey();
-                    ValueDifference<ReservedEntry> valueDiff = entry.getValue();
-                    diffMessage.append(
-                        String.format(
-                            "Domain label %s has entry %s in Datastore and entry"
-                                + " %s in Cloud SQL.\n",
-                            label, valueDiff.leftValue(), valueDiff.rightValue()));
-                  });
+                  (label, valueDiff) ->
+                      diffMessage.append(
+                          String.format(
+                              "Domain label %s has entry %s in Datastore and entry"
+                                  + " %s in Cloud SQL.\n",
+                              label, valueDiff.leftValue(), valueDiff.rightValue())));
          logger.atWarning().log(diffMessage.toString());
        }
      }
@@ -30,6 +30,7 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.domain.Period;
 import google.registry.model.eppcommon.Trid;
+import google.registry.persistence.VKey;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeOverride;
@@ -184,6 +185,10 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
  @Transient // domain-specific
  Set<DomainTransactionRecord> domainTransactionRecords;

+  public Long getId() {
+    return id;
+  }
+
  public Key<? extends EppResource> getParent() {
    return parent;
  }
@@ -234,6 +239,19 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
    return nullToEmptyImmutableCopy(domainTransactionRecords);
  }

+  public static VKey<HistoryEntry> createVKey(Key<HistoryEntry> key) {
+    // TODO(b/159207551): This will likely need some revision.  As it stands, this method was
+    // introduced purely to facilitate testing of VKey specialization in VKeyTranslatorFactory.
+    // This class will likely require that functionality, though perhaps not this implementation of
+    // it.
+    // For now, just assume that the primary key of a history entry is comprised of the parent
+    // type, key and the object identifer.
+    return VKey.create(
+        HistoryEntry.class,
+        key.getParent().getKind() + "/" + key.getParent().getName() + "/" + key.getId(),
+        key);
+  }
+
  @Override
  public Builder asBuilder() {
    return new Builder(clone(this));
@@ -14,18 +14,20 @@

 package google.registry.model.reporting;

+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.util.CollectionUtils.isNullOrEmpty;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import google.registry.util.DomainNameUtils;
+import java.util.Set;
 import javax.persistence.Column;
 import javax.persistence.Entity;
-import javax.persistence.EnumType;
-import javax.persistence.Enumerated;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
@@ -36,11 +38,11 @@ import org.joda.time.LocalDate;
@Entity
@Table(
    indexes = {
-      @Index(name = "safebrowsing_threat_registrar_id_idx", columnList = "registrarId"),
-      @Index(name = "safebrowsing_threat_tld_idx", columnList = "tld"),
-      @Index(name = "safebrowsing_threat_check_date_idx", columnList = "checkDate")
+      @Index(name = "spec11threatmatch_registrar_id_idx", columnList = "registrarId"),
+      @Index(name = "spec11threatmatch_tld_idx", columnList = "tld"),
+      @Index(name = "spec11threatmatch_check_date_idx", columnList = "checkDate")
    })
-public class SafeBrowsingThreat extends ImmutableObject implements Buildable, SqlEntity {
+public class Spec11ThreatMatch extends ImmutableObject implements Buildable, SqlEntity {

  /** The type of threat detected. */
  public enum ThreatType {
@@ -60,10 +62,9 @@ public class SafeBrowsingThreat extends ImmutableObject implements Buildable, Sq
  @Column(nullable = false)
  String domainName;

-  /** The type of threat detected. */
+  /** The types of threat detected. */
  @Column(nullable = false)
-  @Enumerated(EnumType.STRING)
-  ThreatType threatType;
+  Set<ThreatType> threatTypes;

  /** Primary key of the domain table and unique identifier for all EPP resources. */
  @Column(nullable = false)
@@ -89,8 +90,8 @@ public class SafeBrowsingThreat extends ImmutableObject implements Buildable, Sq
    return domainName;
  }

-  public ThreatType getThreatType() {
-    return threatType;
+  public ImmutableSet<ThreatType> getThreatTypes() {
+    return ImmutableSet.copyOf(threatTypes);
  }

  public String getDomainRepoId() {
@@ -119,18 +120,18 @@ public class SafeBrowsingThreat extends ImmutableObject implements Buildable, Sq
    return new Builder(clone(this));
  }

-  /** A builder for constructing {@link SafeBrowsingThreat}, since it is immutable. */
-  public static class Builder extends Buildable.Builder<SafeBrowsingThreat> {
+  /** A builder for constructing {@link Spec11ThreatMatch}, since it is immutable. */
+  public static class Builder extends Buildable.Builder<Spec11ThreatMatch> {
    public Builder() {}

-    private Builder(SafeBrowsingThreat instance) {
+    private Builder(Spec11ThreatMatch instance) {
      super(instance);
    }

    @Override
-    public SafeBrowsingThreat build() {
+    public Spec11ThreatMatch build() {
      checkArgumentNotNull(getInstance().domainName, "Domain name cannot be null");
-      checkArgumentNotNull(getInstance().threatType, "Threat type cannot be null");
+      checkArgumentNotNull(getInstance().threatTypes, "Threat types cannot be null");
      checkArgumentNotNull(getInstance().domainRepoId, "Repo ID cannot be null");
      checkArgumentNotNull(getInstance().registrarId, "Registrar ID cannot be null");
      checkArgumentNotNull(getInstance().checkDate, "Check date cannot be null");
@@ -145,8 +146,10 @@ public class SafeBrowsingThreat extends ImmutableObject implements Buildable, Sq
      return this;
    }

-    public Builder setThreatType(ThreatType threatType) {
-      getInstance().threatType = threatType;
+    public Builder setThreatTypes(ImmutableSet<ThreatType> threatTypes) {
+      checkArgument(!isNullOrEmpty(threatTypes), "threatTypes cannot be null or empty.");
+
+      getInstance().threatTypes = threatTypes;
      return this;
    }

@@ -67,9 +67,17 @@ public class VKeyTranslatorFactory extends AbstractSimpleTranslatorFactory<VKey,
          clazz.getDeclaredMethod("createVKey", com.googlecode.objectify.Key.class);
      return (VKey<T>) createVKeyMethod.invoke(null, new Object[] {key});
    } catch (NoSuchMethodException e) {
-      // Revert to an ofy vkey for now.  TODO(mmuller): remove this when all classes with VKeys have
-      // converters.
-      return VKey.createOfy(clazz, key);
+      checkArgument(
+          key.getParent() == null,
+          "Cannot auto-convert key %s of kind %s because it has a parent.  Add a createVKey(Key) "
+              + "method for it.",
+          key,
+          key.getKind());
+      if (key.getName() != null) {
+        return VKey.create(clazz, key.getName(), key);
+      } else {
+        return VKey.create(clazz, key.getId(), key);
+      }
    } catch (IllegalAccessException | InvocationTargetException e) {
      // If we have a createVKey(Key) method with incorrect permissions or that is non-static, this
      // is probably an error so let's reported.
@@ -58,9 +58,10 @@ public class PersistenceModule {
  public static final String HIKARI_DS_CLOUD_SQL_INSTANCE =
      "hibernate.hikari.dataSource.cloudSqlInstance";

+  @VisibleForTesting
  @Provides
  @DefaultHibernateConfigs
-  public static ImmutableMap<String, String> providesDefaultDatabaseConfigs() {
+  public static ImmutableMap<String, String> provideDefaultDatabaseConfigs() {
    ImmutableMap.Builder<String, String> properties = ImmutableMap.builder();

    properties.put(Environment.DRIVER, "org.postgresql.Driver");
@@ -89,10 +90,25 @@ public class PersistenceModule {
  @Provides
  @Singleton
  @PartialCloudSqlConfigs
-  public static ImmutableMap<String, String> providesPartialCloudSqlConfigs(
+  static ImmutableMap<String, String> providePartialCloudSqlConfigs(
      @Config("cloudSqlJdbcUrl") String jdbcUrl,
      @Config("cloudSqlInstanceConnectionName") String instanceConnectionName,
      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs) {
+    return createPartialSqlConfigs(jdbcUrl, instanceConnectionName, defaultConfigs);
+  }
+
+  @Provides
+  @Singleton
+  @BeamPipelineCloudSqlConfigs
+  static ImmutableMap<String, String> provideBeamPipelineCloudSqlConfigs(
+      @Config("beamCloudSqlJdbcUrl") String jdbcUrl,
+      @Config("beamCloudSqlInstanceConnectionName") String instanceConnectionName,
+      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs) {
+    return createPartialSqlConfigs(jdbcUrl, instanceConnectionName, defaultConfigs);
+  }
+
+  private static ImmutableMap<String, String> createPartialSqlConfigs(
+      String jdbcUrl, String instanceConnectionName, ImmutableMap<String, String> defaultConfigs) {
    HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
    overrides.put(Environment.URL, jdbcUrl);
    overrides.put(HIKARI_DS_SOCKET_FACTORY, "com.google.cloud.sql.postgres.SocketFactory");
@@ -103,7 +119,7 @@ public class PersistenceModule {
  @Provides
  @Singleton
  @AppEngineJpaTm
-  public static JpaTransactionManager providesAppEngineJpaTm(
+  static JpaTransactionManager provideAppEngineJpaTm(
      @Config("cloudSqlUsername") String username,
      KmsKeyring kmsKeyring,
      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
@@ -117,7 +133,7 @@ public class PersistenceModule {
  @Provides
  @Singleton
  @NomulusToolJpaTm
-  public static JpaTransactionManager providesNomulusToolJpaTm(
+  static JpaTransactionManager provideNomulusToolJpaTm(
      @Config("toolsCloudSqlUsername") String username,
      KmsKeyring kmsKeyring,
      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
@@ -130,9 +146,41 @@ public class PersistenceModule {
    return new JpaTransactionManagerImpl(create(overrides), clock);
  }

+  @Provides
+  @Singleton
+  @SocketFactoryJpaTm
+  static JpaTransactionManager provideSocketFactoryJpaTm(
+      @Config("beamCloudSqlUsername") String username,
+      @Config("beamCloudSqlPassword") String password,
+      @Config("beamHibernateHikariMaximumPoolSize") int hikariMaximumPoolSize,
+      @BeamPipelineCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
+      Clock clock) {
+    HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
+    overrides.put(Environment.USER, username);
+    overrides.put(Environment.PASS, password);
+    overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(hikariMaximumPoolSize));
+    return new JpaTransactionManagerImpl(create(overrides), clock);
+  }
+
+  @Provides
+  @Singleton
+  @JdbcJpaTm
+  static JpaTransactionManager provideLocalJpaTm(
+      @Config("beamCloudSqlJdbcUrl") String jdbcUrl,
+      @Config("beamCloudSqlUsername") String username,
+      @Config("beamCloudSqlPassword") String password,
+      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs,
+      Clock clock) {
+    HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
+    overrides.put(Environment.URL, jdbcUrl);
+    overrides.put(Environment.USER, username);
+    overrides.put(Environment.PASS, password);
+    return new JpaTransactionManagerImpl(create(overrides), clock);
+  }
+
  /** Constructs the {@link EntityManagerFactory} instance. */
  @VisibleForTesting
-  public static EntityManagerFactory create(
+  static EntityManagerFactory create(
      String jdbcUrl, String username, String password, ImmutableMap<String, String> configs) {
    HashMap<String, String> properties = Maps.newHashMap(configs);
    properties.put(Environment.URL, jdbcUrl);
@@ -165,11 +213,33 @@ public class PersistenceModule {
  @Documented
  public @interface NomulusToolJpaTm {}

+  /**
+   * Dagger qualifier for {@link JpaTransactionManager} that accesses Cloud SQL using socket
+   * factory. This is meant for applications not running on AppEngine, therefore without access to a
+   * {@link google.registry.keyring.api.Keyring}.
+   */
+  @Qualifier
+  @Documented
+  public @interface SocketFactoryJpaTm {}
+
+  /**
+   * Dagger qualifier for {@link JpaTransactionManager} backed by plain JDBC connections. This is
+   * mainly used by tests.
+   */
+  @Qualifier
+  @Documented
+  public @interface JdbcJpaTm {}
+
  /** Dagger qualifier for the partial Cloud SQL configs. */
  @Qualifier
  @Documented
  @interface PartialCloudSqlConfigs {}

+  /** Dagger qualifier for the Cloud SQL configs used by Beam pipelines. */
+  @Qualifier
+  @Documented
+  @interface BeamPipelineCloudSqlConfigs {}
+
  /** Dagger qualifier for the default Hibernate configurations. */
  // TODO(shicong): Change annotations in this class to none public or put them in a top level
  //  package
@@ -19,6 +19,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

 import com.googlecode.objectify.Key;
 import google.registry.model.ImmutableObject;
+import google.registry.model.translators.VKeyTranslatorFactory;
 import java.io.Serializable;
 import java.util.Optional;

@@ -46,38 +47,17 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    this.primaryKey = primaryKey;
  }

-  /** Creates a {@link VKey} which only contains the sql primary key. */
+  /**
+   * Creates a {@link VKey} which only contains the sql primary key.
+   *
+   * <p>Deprecated. Create symmetric keys with create() instead.
+   */
  public static <T> VKey<T> createSql(Class<? extends T> kind, Object sqlKey) {
    checkArgumentNotNull(kind, "kind must not be null");
    checkArgumentNotNull(sqlKey, "sqlKey must not be null");
    return new VKey(kind, null, sqlKey);
  }

-  /** Creates a {@link VKey} which only contains the ofy primary key. */
-  public static <T> VKey<T> createOfy(
-      Class<? extends T> kind, com.googlecode.objectify.Key<? extends T> ofyKey) {
-    checkArgumentNotNull(kind, "kind must not be null");
-    checkArgumentNotNull(ofyKey, "ofyKey must not be null");
-    return new VKey(kind, ofyKey, null);
-  }
-
-  /**
-   * Creates a {@link VKey} which only contains the ofy primary key by specifying the id of the
-   * {@link Key}.
-   */
-  public static <T> VKey<T> createOfy(Class<? extends T> kind, long id) {
-    return createOfy(kind, Key.create(kind, id));
-  }
-
-  /**
-   * Creates a {@link VKey} which only contains the ofy primary key by specifying the name of the
-   * {@link Key}.
-   */
-  public static <T> VKey<T> createOfy(Class<? extends T> kind, String name) {
-    checkArgumentNotNull(kind, "name must not be null");
-    return createOfy(kind, Key.create(kind, name));
-  }
-
  /** Creates a {@link VKey} which only contains both sql and ofy primary key. */
  public static <T> VKey<T> create(
      Class<? extends T> kind, Object sqlKey, com.googlecode.objectify.Key ofyKey) {
@@ -87,6 +67,11 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    return new VKey(kind, ofyKey, sqlKey);
  }

+  /** Creates a symmetric {@link VKey} in which both sql and ofy keys are {@code id}. */
+  public static <T> VKey<T> create(Class<? extends T> kind, long id) {
+    return new VKey(kind, Key.create(kind, id), id);
+  }
+
  /** Returns the type of the entity. */
  public Class<? extends T> getKind() {
    return this.kind;
@@ -113,4 +98,9 @@ public class VKey<T> extends ImmutableObject implements Serializable {
  public Optional<com.googlecode.objectify.Key<T>> maybeGetOfyKey() {
    return Optional.ofNullable(this.ofyKey);
  }
+
+  /** Convenience method to construct a VKey from an objectify Key. */
+  public static <T> VKey<T> from(Key<T> key) {
+    return VKeyTranslatorFactory.createVKey(key);
+  }
 }
@@ -0,0 +1,34 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA {@link AttributeConverter} for storing/retrieving {@code Set}. */
+@Converter(autoApply = true)
+public class Spec11ThreatMatchThreatTypeSetConverter extends StringSetConverterBase<ThreatType> {
+
+  @Override
+  String toString(ThreatType element) {
+    return element.name();
+  }
+
+  @Override
+  ThreatType fromString(String value) {
+    return ThreatType.valueOf(value);
+  }
+}
@@ -14,7 +14,7 @@

 package google.registry.persistence.converter;

-import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static google.registry.util.CollectionUtils.entriesToImmutableMap;

 import google.registry.persistence.converter.StringMapDescriptor.StringMap;
 import java.util.Map;
@@ -38,7 +38,7 @@ public abstract class StringMapConverterBase<K, V>
        : StringMap.create(
            attribute.entrySet().stream()
                .map(this::convertToDatabaseMapEntry)
-                .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
+                .collect(entriesToImmutableMap()));
  }

  @Override
@@ -47,6 +47,6 @@ public abstract class StringMapConverterBase<K, V>
        ? null
        : dbData.getMap().entrySet().stream()
            .map(this::convertToEntityMapEntry)
-            .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+            .collect(entriesToImmutableMap());
  }
 }
@@ -14,8 +14,9 @@

 package google.registry.persistence.converter;

-import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static google.registry.util.CollectionUtils.entriesToImmutableMap;

+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
@@ -45,7 +46,7 @@ public abstract class TimedTransitionPropertyConverterBase<K, V extends TimedTra
        : StringMap.create(
            attribute.entrySet().stream()
                .map(this::convertToDatabaseMapEntry)
-                .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
+                .collect(entriesToImmutableMap()));
  }

  @Override
@@ -53,10 +54,10 @@ public abstract class TimedTransitionPropertyConverterBase<K, V extends TimedTra
    if (dbData == null) {
      return null;
    }
-    Map<DateTime, K> map =
+    ImmutableMap<DateTime, K> map =
        dbData.getMap().entrySet().stream()
            .map(this::convertToEntityMapEntry)
-            .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+            .collect(entriesToImmutableMap());
    return TimedTransitionProperty.fromValueMap(
        ImmutableSortedMap.copyOf(map), getTimedTransitionSubclass());
  }
@@ -15,18 +15,21 @@
 package google.registry.persistence.transaction;

 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+import static java.util.AbstractMap.SimpleEntry;
 import static java.util.stream.Collectors.joining;

 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import google.registry.persistence.VKey;
 import google.registry.util.Clock;
 import java.lang.reflect.Field;
+import java.util.Map.Entry;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
@@ -98,7 +101,8 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
      T result = work.get();
      txn.commit();
      return result;
-    } catch (RuntimeException e) {
+    } catch (RuntimeException | Error e) {
+      // Error is unchecked!
      try {
        txn.rollback();
        logger.atWarning().log("Error during transaction; transaction rolled back");
@@ -253,20 +257,18 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
  }

  @Override
-  public <T> ImmutableList<T> load(Iterable<VKey<T>> keys) {
+  public <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys) {
    checkArgumentNotNull(keys, "keys must be specified");
    assertInTransaction();
    return StreamSupport.stream(keys.spliterator(), false)
+        // Accept duplicate keys.
+        .distinct()
        .map(
-            key -> {
-              T entity = getEntityManager().find(key.getKind(), key.getSqlKey());
-              if (entity == null) {
-                throw new NoSuchElementException(
-                    key.getKind().getName() + " with key " + key.getSqlKey() + " not found.");
-              }
-              return entity;
-            })
-        .collect(toImmutableList());
+            key ->
+                new SimpleEntry<VKey<? extends T>, T>(
+                    key, getEntityManager().find(key.getKind(), key.getSqlKey())))
+        .filter(entry -> entry.getValue() != null)
+        .collect(toImmutableMap(Entry::getKey, Entry::getValue));
  }

  @Override
@@ -16,6 +16,7 @@ package google.registry.persistence.transaction;

 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import google.registry.persistence.VKey;
 import java.util.NoSuchElementException;
 import java.util.Optional;
@@ -119,7 +120,7 @@ public interface TransactionManager {
   *
   * @throws NoSuchElementException if any of the keys are not found.
   */
-  <T> ImmutableList<T> load(Iterable<VKey<T>> keys);
+  <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys);

  /** Loads all entities of the given type, returns empty if there is no such entity. */
  <T> ImmutableList<T> loadAll(Class<T> clazz);
@@ -14,12 +14,17 @@

 package google.registry.persistence.transaction;

+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
 import com.google.appengine.api.utils.SystemProperty;
 import com.google.appengine.api.utils.SystemProperty.Environment.Value;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Suppliers;
+import google.registry.config.RegistryEnvironment;
 import google.registry.model.ofy.DatastoreTransactionManager;
 import google.registry.persistence.DaggerPersistenceComponent;
+import google.registry.tools.RegistryToolEnvironment;
 import google.registry.util.NonFinalForTesting;
 import java.util.function.Supplier;

@@ -82,8 +87,13 @@ public class TransactionManagerFactory {
  }

  /** Sets the return of {@link #jpaTm()} to the given instance of {@link JpaTransactionManager}. */
-  public static void setJpaTm(JpaTransactionManager newJpaTm) {
-    jpaTm = Suppliers.ofInstance(newJpaTm);
+  public static void setJpaTm(Supplier<JpaTransactionManager> jpaTmSupplier) {
+    checkNotNull(jpaTmSupplier, "jpaTmSupplier");
+    checkState(
+        RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
+            || RegistryToolEnvironment.get() != null,
+        "setJpamTm() should only be called by tools and tests.");
+    jpaTm = jpaTmSupplier;
  }

  /** Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}. */
@@ -22,7 +22,6 @@ import google.registry.model.pricing.PremiumPricingEngine;
 import google.registry.model.pricing.PremiumPricingEngine.DomainPrices;
 import google.registry.model.registry.Registry;
 import java.util.Map;
-import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;

@@ -52,11 +51,6 @@ public final class PricingEngineProxy {
    return getPricesForDomainName(domainName, priceTime).isPremium();
  }

-  /** Returns the fee class of the specified domain name. */
-  public static Optional<String> getDomainFeeClass(String domainName, DateTime priceTime) {
-    return getPricesForDomainName(domainName, priceTime).getFeeClass();
-  }
-
  /**
   * Returns the full {@link DomainPrices} details for the given domain name by dispatching to the
   * appropriate {@link PremiumPricingEngine} based on what is configured for the TLD that the
@@ -295,7 +295,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
      query = query.filter("currentSponsorClientId", desiredRegistrar.get());
    }
    return StreamSupport.stream(query.keys().spliterator(), false)
-        .map(key -> VKey.createOfy(HostResource.class, key))
+        .map(key -> VKey.from(key))
        .collect(toImmutableSet());
  }

@@ -406,7 +406,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
    }
    return searchByNameserverRefs(
        StreamSupport.stream(query.keys().spliterator(), false)
-            .map(key -> VKey.createOfy(HostResource.class, key))
+            .map(key -> VKey.from(key))
            .collect(toImmutableSet()));
  }

@@ -342,7 +342,7 @@ public class RdapJsonFormatter {
    // Kick off the database loads of the nameservers that we will need, so it can load
    // asynchronously while we load and process the contacts.
    ImmutableSet<HostResource> loadedHosts =
-        ImmutableSet.copyOf(tm().load(domainBase.getNameservers()));
+        ImmutableSet.copyOf(tm().load(domainBase.getNameservers()).values());
    // Load the registrant and other contacts and add them to the data.
    Map<Key<ContactResource>, ContactResource> loadedContacts =
        ofy()
@@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSetMultimap;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.googlecode.objectify.Result;
 import google.registry.model.EppResource;
 import google.registry.model.contact.ContactResource;
@@ -44,6 +45,11 @@ public final class RdeStagingMapper extends Mapper<EppResource, PendingDeposit,

  private static final long serialVersionUID = -1518185703789372524L;

+  // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
+  // if sneaks into production we would get an extra signal.
+  private static final ImmutableSet<Registrar.Type> IGNORED_REGISTRAR_TYPES =
+      Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
+
  private final RdeMarshaller marshaller;
  private final ImmutableSetMultimap<String, PendingDeposit> pendings;

@@ -64,6 +70,9 @@ public final class RdeStagingMapper extends Mapper<EppResource, PendingDeposit,
    if (resource == null) {
      long registrarsEmitted = 0;
      for (Registrar registrar : Registrar.loadAllCached()) {
+        if (IGNORED_REGISTRAR_TYPES.contains(registrar.getType())) {
+          continue;
+        }
        DepositFragment fragment = marshaller.marshalRegistrar(registrar);
        for (PendingDeposit pending : pendings.values()) {
          emit(pending, fragment);
@@ -15,7 +15,6 @@
 package google.registry.reporting.icann;

 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -27,6 +26,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteStreams;
 import com.googlecode.objectify.Key;
@@ -209,9 +209,9 @@ public final class IcannReportingUploadAction implements Runnable {

    ImmutableSet<Registry> registries = Registries.getTldEntitiesOfType(TldType.REAL);

-    Map<Key<Cursor>, Registry> activityKeyMap =
+    ImmutableMap<Key<Cursor>, Registry> activityKeyMap =
        loadKeyMap(registries, CursorType.ICANN_UPLOAD_ACTIVITY);
-    Map<Key<Cursor>, Registry> transactionKeyMap =
+    ImmutableMap<Key<Cursor>, Registry> transactionKeyMap =
        loadKeyMap(registries, CursorType.ICANN_UPLOAD_TX);

    ImmutableSet.Builder<Key<Cursor>> keys = new ImmutableSet.Builder<>();
@@ -229,9 +229,9 @@ public final class IcannReportingUploadAction implements Runnable {
    return cursors.build();
  }

-  private Map<Key<Cursor>, Registry> loadKeyMap(
+  private ImmutableMap<Key<Cursor>, Registry> loadKeyMap(
      ImmutableSet<Registry> registries, CursorType type) {
-    return registries.stream().collect(toImmutableMap(r -> Cursor.createKey(type, r), r -> r));
+    return Maps.uniqueIndex(registries, r -> Cursor.createKey(type, r));
  }

  /**
@@ -15,7 +15,6 @@
 package google.registry.schema.cursor;

 import static com.google.appengine.api.search.checkers.Preconditions.checkNotNull;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -23,6 +22,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
 import com.google.common.flogger.FluentLogger;
 import google.registry.model.common.Cursor.CursorType;
 import google.registry.schema.cursor.Cursor.CursorId;
@@ -171,11 +171,11 @@ public class CursorDao {
      // Load all the cursors of that type from Cloud SQL
      List<Cursor> cloudSqlCursors = loadByType(type);

-      // Create a map of each tld to its cursor if one exists
+      // Create a map of each TLD to its cursor if one exists.
      ImmutableMap<String, Cursor> cloudSqlCursorMap =
-          cloudSqlCursors.stream().collect(toImmutableMap(c -> c.getScope(), c -> c));
+          Maps.uniqueIndex(cloudSqlCursors, Cursor::getScope);

-      // Compare each Datastore cursor with its corresponding Cloud SQL cursor
+      // Compare each Datastore cursor with its corresponding Cloud SQL cursor.
      for (google.registry.model.common.Cursor cursor : cursors.keySet()) {
        Cursor cloudSqlCursor = cloudSqlCursorMap.get(cursors.get(cursor));
        compare(cursor, cloudSqlCursor, cursors.get(cursor));
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.cmd.QueryKeys;
+import google.registry.flows.poll.PollFlowUtils;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.poll.PollMessage.Autorenew;
 import google.registry.model.poll.PollMessage.OneTime;
@@ -49,8 +50,7 @@ import javax.inject.Inject;
 * delete confirmations) and it is desired that the registrar be able to ACK the rest of the poll
 * messages in-band.
 *
- * <p>This command only ACKs {@link OneTime} poll messages because that's our only use case so far,
- * but it could be extended to ACK {@link Autorenew} poll messages as well if needed. The main
+ * <p>This command ACKs both {@link OneTime} and {@link Autorenew} poll messages. The main
 * difference is that one-time poll messages are deleted when ACKed whereas Autorenews are sometimes
 * modified and re-saved instead, if the corresponding domain is still active.
 *
@@ -85,18 +85,17 @@ final class AckPollMessagesCommand implements CommandWithRemoteApi {
  @Override
  public void run() {
    QueryKeys<PollMessage> query = getPollMessagesQuery(clientId, clock.nowUtc()).keys();
+    // TODO(b/160325686): Remove the batch logic after db migration.
    for (List<Key<PollMessage>> keys : Iterables.partition(query, BATCH_SIZE)) {
-      tm()
-          .transact(
+      tm().transact(
              () -> {
                // Load poll messages and filter to just those of interest.
                ImmutableList<PollMessage> pollMessages =
                    ofy().load().keys(keys).values().stream()
-                        .filter(pm -> pm instanceof OneTime)
                        .filter(pm -> isNullOrEmpty(message) || pm.getMsg().contains(message))
                        .collect(toImmutableList());
                if (!dryRun) {
-                  ofy().delete().entities(pollMessages).now();
+                  pollMessages.forEach(PollFlowUtils::ackPollMessage);
                }
                pollMessages.forEach(
                    pm ->
@@ -50,7 +50,6 @@ import java.lang.annotation.RetentionPolicy;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import javax.annotation.Nullable;
-import javax.inject.Named;
 import javax.inject.Qualifier;
 import javax.inject.Singleton;

@@ -151,10 +150,10 @@ public class AuthModule {
  public static String provideLocalCredentialJson(
      Lazy<GoogleClientSecrets> clientSecrets,
      @StoredCredential Lazy<Credential> credential,
-      @Nullable @Named("credentialFileName") String credentialFilename) {
+      @Nullable @Config("credentialFilePath") String credentialFilePath) {
    try {
-      if (credentialFilename != null) {
-        return new String(Files.readAllBytes(Paths.get(credentialFilename)), UTF_8);
+      if (credentialFilePath != null) {
+        return new String(Files.readAllBytes(Paths.get(credentialFilePath)), UTF_8);
      } else {
        return new Gson()
            .toJson(
@@ -116,6 +116,13 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
      description = "Tld type (REAL or TEST)")
  private TldType tldType;

+  @Nullable
+  @Parameter(
+      names = "--disable_invoicing",
+      description = "Whether invoicing is disabled for a REAL tld.",
+      arity = 1)
+  private Boolean disableInvoicing;
+
  @Nullable
  @Parameter(
      names = "--create_billing_cost",
@@ -320,6 +327,7 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
      Optional.ofNullable(serverStatusChangeCost)
          .ifPresent(builder::setServerStatusChangeBillingCost);
      Optional.ofNullable(tldType).ifPresent(builder::setTldType);
+      Optional.ofNullable(disableInvoicing).ifPresent(builder::setDisableInvoicing);
      Optional.ofNullable(lordnUsername).ifPresent(u -> builder.setLordnUsername(u.orElse(null)));
      Optional.ofNullable(claimsPeriodEnd).ifPresent(builder::setClaimsPeriodEnd);
      Optional.ofNullable(numDnsPublishShards).ifPresent(builder::setNumDnsPublishLocks);
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import com.googlecode.objectify.Key;
 import google.registry.batch.AsyncTaskEnqueuer;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.domain.DomainBase;
@@ -52,13 +53,16 @@ public final class DomainLockUtils {
  private static final int VERIFICATION_CODE_LENGTH = 32;

  private final StringGenerator stringGenerator;
+  private final String registryAdminRegistrarId;
  private final AsyncTaskEnqueuer asyncTaskEnqueuer;

  @Inject
  public DomainLockUtils(
      @Named("base58StringGenerator") StringGenerator stringGenerator,
+      @Config("registryAdminClientId") String registryAdminRegistrarId,
      AsyncTaskEnqueuer asyncTaskEnqueuer) {
    this.stringGenerator = stringGenerator;
+    this.registryAdminRegistrarId = registryAdminRegistrarId;
    this.asyncTaskEnqueuer = asyncTaskEnqueuer;
  }

@@ -217,7 +221,7 @@ public final class DomainLockUtils {
  private RegistryLock.Builder createLockBuilder(
      String domainName, String registrarId, @Nullable String registrarPocId, boolean isAdmin) {
    DateTime now = jpaTm().getTransactionTime();
-    DomainBase domainBase = getDomain(domainName, now);
+    DomainBase domainBase = getDomain(domainName, registrarId, now);
    verifyDomainNotLocked(domainBase);

    // Multiple pending actions are not allowed
@@ -242,7 +246,7 @@ public final class DomainLockUtils {
  private RegistryLock.Builder createUnlockBuilder(
      String domainName, String registrarId, boolean isAdmin, Optional<Duration> relockDuration) {
    DateTime now = jpaTm().getTransactionTime();
-    DomainBase domainBase = getDomain(domainName, now);
+    DomainBase domainBase = getDomain(domainName, registrarId, now);
    Optional<RegistryLock> lockOptional =
        RegistryLockDao.getMostRecentVerifiedLockByRepoId(domainBase.getRepoId());

@@ -303,10 +307,19 @@ public final class DomainLockUtils {
        domainBase.getDomainName());
  }

-  private static DomainBase getDomain(String domainName, DateTime now) {
-    return loadByForeignKeyCached(DomainBase.class, domainName, now)
-        .orElseThrow(
-            () -> new IllegalArgumentException(String.format("Unknown domain %s", domainName)));
+  private DomainBase getDomain(String domainName, String registrarId, DateTime now) {
+    DomainBase domain =
+        loadByForeignKeyCached(DomainBase.class, domainName, now)
+            .orElseThrow(
+                () -> new IllegalArgumentException(String.format("Unknown domain %s", domainName)));
+    // The user must have specified either the correct registrar ID or the admin registrar ID
+    checkArgument(
+        registryAdminRegistrarId.equals(registrarId)
+            || domain.getCurrentSponsorClientId().equals(registrarId),
+        "Domain %s is not owned by registrar %s",
+        domainName,
+        registrarId);
+    return domain;
  }

  private static RegistryLock getByVerificationCode(String verificationCode) {
@@ -317,8 +330,8 @@ public final class DomainLockUtils {
                    String.format("Invalid verification code %s", verificationCode)));
  }

-  private static void applyLockStatuses(RegistryLock lock, DateTime lockTime) {
-    DomainBase domain = getDomain(lock.getDomainName(), lockTime);
+  private void applyLockStatuses(RegistryLock lock, DateTime lockTime) {
+    DomainBase domain = getDomain(lock.getDomainName(), lock.getRegistrarId(), lockTime);
    verifyDomainNotLocked(domain);

    DomainBase newDomain =
@@ -330,8 +343,8 @@ public final class DomainLockUtils {
    saveEntities(newDomain, lock, lockTime, true);
  }

-  private static void removeLockStatuses(RegistryLock lock, boolean isAdmin, DateTime unlockTime) {
-    DomainBase domain = getDomain(lock.getDomainName(), unlockTime);
+  private void removeLockStatuses(RegistryLock lock, boolean isAdmin, DateTime unlockTime) {
+    DomainBase domain = getDomain(lock.getDomainName(), lock.getRegistrarId(), unlockTime);
    if (!isAdmin) {
      verifyDomainLocked(domain);
    }
@@ -29,6 +29,7 @@ import com.google.appengine.tools.remoteapi.RemoteApiOptions;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
+import google.registry.beam.initsql.BeamJpaModule;
 import google.registry.config.RegistryConfig;
 import google.registry.model.ofy.ObjectifyService;
 import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -153,11 +154,15 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
      return;
    }

-    checkState(RegistryToolEnvironment.get() == environment,
+    checkState(
+        RegistryToolEnvironment.get() == environment,
        "RegistryToolEnvironment argument pre-processing kludge failed.");

    component =
-        DaggerRegistryToolComponent.builder().credentialFilename(credentialJson).build();
+        DaggerRegistryToolComponent.builder()
+            .credentialFilePath(credentialJson)
+            .beamJpaModule(new BeamJpaModule(credentialJson))
+            .build();

    // JCommander stores sub-commands as nested JCommander objects containing a list of user objects
    // to be populated.  Extract the subcommand by getting the JCommander wrapper and then
@@ -240,7 +245,7 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
      // Enable Cloud SQL for command that needs remote API as they will very likely use
      // Cloud SQL after the database migration. Note that the DB password is stored in Datastore
      // and it is already initialized above.
-      TransactionManagerFactory.setJpaTm(component.nomulusToolJpaTransactionManager());
+      TransactionManagerFactory.setJpaTm(() -> component.nomulusToolJpaTransactionManager().get());
    }

    command.run();
@@ -16,9 +16,12 @@ package google.registry.tools;

 import dagger.BindsInstance;
 import dagger.Component;
+import dagger.Lazy;
 import google.registry.batch.BatchModule;
+import google.registry.beam.initsql.BeamJpaModule;
 import google.registry.bigquery.BigqueryModule;
 import google.registry.config.CredentialModule.LocalCredentialJson;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.dns.writer.VoidDnsWriterModule;
 import google.registry.dns.writer.clouddns.CloudDnsWriterModule;
@@ -41,7 +44,6 @@ import google.registry.tools.AuthModule.LocalCredentialModule;
 import google.registry.util.UtilsModule;
 import google.registry.whois.WhoisModule;
 import javax.annotation.Nullable;
-import javax.inject.Named;
 import javax.inject.Singleton;

 /**
@@ -56,6 +58,7 @@ import javax.inject.Singleton;
      AppEngineAdminApiModule.class,
      AuthModule.class,
      BatchModule.class,
+      BeamJpaModule.class,
      BigqueryModule.class,
      ConfigModule.class,
      CloudDnsWriterModule.class,
@@ -124,12 +127,14 @@ interface RegistryToolComponent {
  String googleCredentialJson();

  @NomulusToolJpaTm
-  JpaTransactionManager nomulusToolJpaTransactionManager();
+  Lazy<JpaTransactionManager> nomulusToolJpaTransactionManager();

  @Component.Builder
  interface Builder {
    @BindsInstance
-    Builder credentialFilename(@Nullable @Named("credentialFileName") String credentialFilename);
+    Builder credentialFilePath(@Nullable @Config("credentialFilePath") String credentialFilePath);
+
+    Builder beamJpaModule(BeamJpaModule beamJpaModule);

    RegistryToolComponent build();
  }
@@ -15,7 +15,6 @@
 package google.registry.tools;

 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.ImmutableSetMultimap.flatteningToImmutableSetMultimap;
 import static google.registry.util.CollectionUtils.nullToEmpty;
@@ -29,6 +28,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Maps;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.request.Action.Service;
@@ -53,8 +53,7 @@ final class SetNumInstancesCommand implements CommandWithRemoteApi {
      ImmutableSet.copyOf(Service.values());

  private static final ImmutableMap<String, Service> SERVICE_ID_TO_SERVICE =
-      ALL_DEPLOYED_SERVICES.stream()
-          .collect(toImmutableMap(service -> service.getServiceId(), service -> service));
+      Maps.uniqueIndex(ALL_DEPLOYED_SERVICES, Service::getServiceId);

  // TODO(b/119629679): Use List<Service> after upgrading jcommander to latest version.
  @Parameter(
@@ -149,7 +149,7 @@ final class UniformRapidSuspensionCommand extends MutatingEppToolCommand {

  private ImmutableSortedSet<String> getExistingNameservers(DomainBase domain) {
    ImmutableSortedSet.Builder<String> nameservers = ImmutableSortedSet.naturalOrder();
-    for (HostResource host : tm().load(domain.getNameservers())) {
+    for (HostResource host : tm().load(domain.getNameservers()).values()) {
      nameservers.add(host.getForeignKey());
    }
    return nameservers.build();
@@ -214,7 +214,7 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
    private void emitForSubordinateHosts(DomainBase domain) {
      ImmutableSet<String> subordinateHosts = domain.getSubordinateHosts();
      if (!subordinateHosts.isEmpty()) {
-        for (HostResource unprojectedHost : tm().load(domain.getNameservers())) {
+        for (HostResource unprojectedHost : tm().load(domain.getNameservers()).values()) {
          HostResource host = loadAtPointInTime(unprojectedHost, exportTime).now();
          // A null means the host was deleted (or not created) at this time.
          if ((host != null) && subordinateHosts.contains(host.getHostName())) {
@@ -283,7 +283,7 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
      Duration dnsDefaultDsTtl) {
    StringBuilder result = new StringBuilder();
    String domainLabel = stripTld(domain.getDomainName(), domain.getTld());
-    for (HostResource nameserver : tm().load(domain.getNameservers())) {
+    for (HostResource nameserver : tm().load(domain.getNameservers()).values()) {
      result.append(
          String.format(
              NS_FORMAT,
@@ -454,7 +454,13 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
                .orElseThrow(
                    () ->
                        new FormException(
-                            "Not allowed to set registry lock password directly on new contact"));
+                            "Cannot set registry lock password directly on new contact"));
+        // Can't modify registry lock email address
+        if (!Objects.equals(
+            updatedContact.getRegistryLockEmailAddress(),
+            existingContact.getRegistryLockEmailAddress())) {
+          throw new FormException("Cannot modify registryLockEmailAddress through the UI");
+        }
        if (updatedContact.isRegistryLockAllowed()) {
          // the password must have been set before or the user was allowed to set it now
          if (!existingContact.isAllowedToSetRegistryLockPassword()
@@ -464,7 +470,8 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
        }
        if (updatedContact.isAllowedToSetRegistryLockPassword()) {
          if (!existingContact.isAllowedToSetRegistryLockPassword()) {
-            throw new FormException("Cannot set isAllowedToSetRegistryLockPassword through UI");
+            throw new FormException(
+                "Cannot modify isAllowedToSetRegistryLockPassword through the UI");
          }
        }
      }
@@ -19,7 +19,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.security.JsonResponseHelper.Status.ERROR;
 import static google.registry.security.JsonResponseHelper.Status.SUCCESS;
-import static google.registry.ui.server.registrar.RegistrarConsoleModule.PARAM_CLIENT_ID;
 import static google.registry.ui.server.registrar.RegistryLockGetAction.getContactMatchingLogin;
 import static google.registry.ui.server.registrar.RegistryLockGetAction.getRegistrarAndVerifyLockAccess;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
@@ -116,10 +115,8 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
      checkArgumentNotNull(input, "Null JSON");
      RegistryLockPostInput postInput =
          GSON.fromJson(GSON.toJsonTree(input), RegistryLockPostInput.class);
-      checkArgument(
-          !Strings.isNullOrEmpty(postInput.clientId),
-          "Missing key for client: %s",
-          PARAM_CLIENT_ID);
+      String registrarId = postInput.registrarId;
+      checkArgument(!Strings.isNullOrEmpty(registrarId), "Missing key for registrarId");
      checkArgument(!Strings.isNullOrEmpty(postInput.domainName), "Missing key for domainName");
      checkNotNull(postInput.isLock, "Missing key for isLock");
      UserAuthInfo userAuthInfo =
@@ -135,12 +132,12 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
                    postInput.isLock
                        ? domainLockUtils.saveNewRegistryLockRequest(
                            postInput.domainName,
-                            postInput.clientId,
+                            registrarId,
                            userEmail,
                            registrarAccessor.isAdmin())
                        : domainLockUtils.saveNewRegistryUnlockRequest(
                            postInput.domainName,
-                            postInput.clientId,
+                            registrarId,
                            registrarAccessor.isAdmin(),
                            Optional.ofNullable(postInput.relockDurationMillis).map(Duration::new));
                sendVerificationEmail(registryLock, userEmail, postInput.isLock);
@@ -190,9 +187,9 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
      return user.getEmail();
    }
    // Verify that the user can access the registrar, that the user has
-    // registry lock enabled, and that the user providjed a correct password
+    // registry lock enabled, and that the user provided a correct password
    Registrar registrar =
-        getRegistrarAndVerifyLockAccess(registrarAccessor, postInput.clientId, false);
+        getRegistrarAndVerifyLockAccess(registrarAccessor, postInput.registrarId, false);
    RegistrarContact registrarContact =
        getContactMatchingLogin(user, registrar)
            .orElseThrow(
@@ -215,7 +212,7 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc

  /** Value class that represents the expected input body from the UI request. */
  private static class RegistryLockPostInput {
-    private String clientId;
+    private String registrarId;
    private String domainName;
    private Boolean isLock;
    private String password;
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Lai JiangandGitHub	e4d0571125	Increase the maximum number of nodes in a nood pool to 15 (#672 )	2020-07-10 21:54:18 -04:00
Ben McIlwainandGitHub	4cb88ab6e7	Convert RDE tests (and some test rules) from JUnit 4 to JUnit 5 (#677 ) * Add JUnit Params and start using it * Convert rest of RDE tests * Don't check headers for generated tests * Expand visibility to fix build breakage * Bump JUnit versions to 5.6.2	2020-07-10 21:32:36 -04:00
gbrodmanandGitHub	987f390ff7	Run 'npm audit fix' to fix low-severity vulnerabilities in packages (#676 )	2020-07-10 15:57:59 -04:00
Ben McIlwainandGitHub	ca756e14e6	Migrate all model tests from JUnit 4 to JUnit 5 (#675 ) * Make first handful of tests JUnit 5 * Migrate rest of model package to JUnit 5	2020-07-10 14:56:28 -04:00
Ben McIlwainandGitHub	caa0cd9d61	Add a "coreDev" gradle target (#667 ) * Add a "buildFmt" gradle target This does the same thing as the automatic Java build target, except instead of failing if the code formatting isn't correct, it just automatically reformats as necessary and continues on. * Remove unnecessary mustRunAfters * Make it run tests too, and add :taskTree task * Rename task to coreDev and remove run afters * Add task tree dependency * Actually that may not be necessary	2020-07-10 10:03:59 -04:00
Legina ChenandGitHub	7806cc7edb	Add domainRepoId to Subdomain class (#674 ) * Change Subdomain class to contain domainRepoId * Remove jpaTm from Spec11PipelineTest and change clientId -> registrarId * Remove 'client' from a comment * Include changes to Spec11Pipeline * add SafeBrowsingTransforms * Run style	2020-07-09 16:26:35 -07:00
Lai JiangandGitHub	0964fdf1dc	Upgrade to Gradle 6.5.1 (#673 )	2020-07-09 14:04:22 -04:00
gbrodmanandGitHub	d17ec1fcb1	Use an enum instead of boolean in EntityTestCase constructor (#669 ) * Use an enum instead of boolean in EntityTestCase constructor It's more clear to use an enum rather than just a simple boolean * Add Javadoc and make the enum name more verbose	2020-07-09 12:54:32 -04:00
Ben McIlwainandGitHub	fac5987c13	Double the # of pubapi instances to better handle traffic spikes (#671 ) * Double the # of pubapi instances to better handle traffic spikes We may also consider switching to an automatic scaling mode soon, on the hope that it's working better than the last time we tried it (it would help to keep resource costs down at least).	2020-07-09 11:52:15 -04:00
Ben McIlwainandGitHub	a3319e0026	Upgrade flow test classes to JUnit 5 (#666 ) Most of the diffs are visibility changes. Also deletes ShardableTestCase, which was only necessary because of Blaze (and possible Bazel) limitations.	2020-07-08 14:08:05 -04:00
Weimin YuandGitHub	5578464e06	Make sure uncommitted txn is rolled back (#665 ) * Make sure uncommit txn is rolled back The try block around commit that catches RuntimeException should also catch Error, which is also unchecked.	2020-07-06 17:39:13 -04:00
gbrodmanandGitHub	c24a61f813	Refactor ContactResource into ContactBase and create ContactHistory (#634 ) * Create ContactHistory class + table This is similar to #587, but with contacts instead of hosts. This also includes a couple cleanups for HostHistoryTest and RegistryLockDaoTest, just making code more proper (we shouldn't be referencing constant revision IDs when using a sequence that is used by multiple classes, and RLDT can extend EntityTest) Note as well that we set ContactHistory to use the same revision ID sequence as HostHistory. * Move ContactResource -> ContactBase * Alter ContactBase and ContactResource	2020-07-06 12:52:16 -04:00
gbrodmanandGitHub	806f3b2456	Verify that the RegistryLock input has the correct registrar ID (#661 ) * Verify that the RegistryLock input has the correct registrar ID We already verify (correctly) that the user has access to the registrar they specify, but nowhere did we verify that the registrar ID they used is actually the current sponsor ID for the domain in question. This is an oversight caused by the fact that our testing framework only uses admin accounts, which by the nature of things have access to all registrars and domains. In addition, rename "clientId" to "registrarId" in the RLPA object * Change the wording on the incorrect-registrar message	2020-07-05 22:31:14 -04:00
gbrodmanandGitHub	333170a724	Allow users the option of seeing their registry lock password (#663 ) * Allow users the option of seeing their registry lock password Only when entering it for the first time, of course.	2020-07-05 20:08:22 -04:00
Lai JiangandGitHub	47eeb8c4e4	Output PO number in detailed report (#659 ) * Output PO number in detailed report The PO number header was added during the beam migration but we forgot to print the actual data in the corresponding column. This resulted in a misalignment of columns in the detailed report. This PR fixes it. Note that we cannot drop PO number from the header (as is not useful in the detailed report) because the header represents all fields that are to be parsed from the SQL query results, and PO number is needed when generating the invoice itself. By dual-purposing the header (both as the required fields in the parser and the first line in the detailed report) we have to include the value of PO number in the detailed report CSV as well.	2020-07-01 19:09:05 -04:00
Shicong HuangandGitHub	391929b518	Expand AckPollMessagesCommand to ack PollMessage.Autorenew (#647 ) * Expand AckPollMessagesCommand to ack PollMessage.Autorenew * Rebase on master and address comment * Resolve comments	2020-07-01 15:06:35 -04:00
gbrodmanandGitHub	7f62b7a89c	Include the registry lock email in the JS object as a sensitive field (#658 ) * Include the registry lock email in the JS object as a sensitive field * Change wording of exceptions to be more consistent	2020-07-01 13:05:21 -04:00
gbrodmanandGitHub	a1da32bfde	Disambiguate injected Cloud SQL parameter names (#657 ) * Disambiguate injected Cloud SQL parameter names This allows us to also inject the BeamJpaModule into RegistryTool, which allows us to use the SocketJpaTransactionManager in Beam pipelines. Some side effects of this include: - duplication of KMS connections -- one standard, one Beam - duplication of the creation of the partial Hibernate SQL configs - removal of ambiguity between credentialFileName, credentialFilename, and credentialFilePath -- we now use the latter. - Performing the credential null check when instantiating the SQL connection rather than when instantiating the module object. See the code comments for more details on this. I verified that this compiles and the tests run successfully when injecting a @SocketFactoryJpaTm into a Beam pipeline. * Remove two unnecessary config points and change the name of two params * Use @Config instead of @Named and change the pool size * Replace non-visible link with code	2020-07-01 11:55:21 -04:00
Weimin YuandGitHub	1961a5759d	Load Datastore snapshot from backup files (#660 ) * Load Datastore snapshot from backup files Defined a composite transform that loads from a Datastore export and concurrent CommitLog files, identify entities that still exist at the end of the time window, and resolve their latest states in the window.	2020-07-01 09:58:42 -04:00
Weimin YuandGitHub	d065ff63fc	Exclude Test/Monitoring Registrars from escrow (#655 ) * Exclude Test/Monitoring Registrars from escrow Registrars used for testing and monitoring should not be included in Data escrow. They also lack the required ianaIdentifier property and would fail ICANN data validation. Note that since alpha and crash environments have bad data that break the RDE process, we need to verify this change in Sandbox.	2020-06-26 19:11:22 -04:00
Michael MullerandGitHub	07ff6279bb	Make EppResource.loadCached() use batched fetch (#652 ) * Make EppResource.loadCached() use batched fetch Use a batched fetch (ofy().load().keys(...)) from datastore in EppResource.loadCached(). To support this, convert TransactionManager.load(Iterable<VKey>) to accept the more flexible generic parameters and return a map. * Simplify datastore key streaming * Changes requested in review.	2020-06-26 13:50:02 -04:00
Legina ChenandGitHub	5c5b6b20ce	Allow multiple threat types in the Spec11ThreatMatch table (#650 ) * Update to generic Spec11ThreatMatch table * Fix SQL syntax * Make changes to the schema and add a test for null and empty threatTypes * Fix a small typo * Change the exception thrown with illegal arguments Change the import for isNullOrEmpty * Fix import for checkArgument * Added a threat to test multiple threat types	2020-06-26 10:35:00 -07:00
Ben McIlwainandGitHub	74b2de5c35	Make ImmutableMap Stream collect()ion nicer (#654 ) This adds an entriesToImmutableMap() collector that can be used in place of toImmutableMap(Map.Entry::getkey, Map.Entry::getValue()). It also fixes up some existing calls that use toImmutableMap() when terser alternatives exist.	2020-06-26 11:57:26 -04:00
Ben McIlwainandGitHub	fba6804d3b	Properly handle restore fees on domain checks (#646 ) * Properly handle restore command fees for domain checks * Get tests working and handle fee classes better * Remove unused ImmutableSet imports * Fix code review concerns, mostly surrounding immutability * Rename more targetIds and make them immutable too * Merge remote-tracking branch 'upstream/master' into domain-check-restore-fees * Fix Javadoc formatting	2020-06-26 10:59:46 -04:00
Lai JiangandGitHub	db5311075d	Patch terraform changes made internally (#651 ) There were several LSC that made some formatting changes to our .tf files. Export these changes externally for consistency.	2020-06-25 13:59:37 -04:00
Ben McIlwainandGitHub	6e26dacdff	Make nomulus compatible with Java 11 (#649 ) * Make nomulus compatible with Java 11 This fixes the double-spacing bug with logged EPP XML on App Engine that started appearing after App Engine switching from using Java 8 to Java 11. Java 9 made some changes to XML Transformer classes that needed a little bit of work to accommodate. This also fixes the unit tests that were failing in Java 11 (all of which were related to said XML Transformer changes). * Make code review changes	2020-06-25 13:17:22 -04:00
Weimin YuandGitHub	2e5466f32f	Fix flaky tests that depends on order of rules SystemPropertyRule in some cases should be applied last: when multiple rules exist and and modified property is checked in cleanups. ConsoleOteSetupActionTest and ConsoleRegistrarCreatorActionTest are two such classes, and can be flaky in JUnit 4. This PR migrates them to JUnit5 and applies ordering to extensions in them. Added a mockito dependency, and upgraded mockito-core to 3.3.3. Meaningful changes: SystemPropertyRule.java and ConsoleOteSetupActionTest.java, and ConsoleRegistrarCreatorActionTest.java	2020-06-24 23:14:47 -04:00
Lai JiangandGitHub	a65a3fd8b7	Add a property in Registry to disable invoicing (#644 ) The added property will by default be null for existing entities and false for new entities. We could write a temporary MapReduce to set it to false for all existing ones, but it seems easier to just modify the query to account for the nullness of the column. The need to check for null won't exist once we migrate the Registry entity to Cloud SQL. TESTED=deployed to alpha, set the property for tld zombo to true and triggered an export, verified that the column is null for all rows other than the row for zombo.	2020-06-24 16:21:38 -04:00
Weimin YuandGitHub	071800ae50	Delay SQL availability check during Tool start (#643 ) * Delay SQL availability check during Tool start During Cloud SQL migration, Cloud SQL password setup in Registry server requires a few tool commands. Such commands must be able to run before Cloud SQL is configured. In this change, we deplay SQL credential loading until access is needed.	2020-06-24 14:00:36 -04:00
Shicong HuangandGitHub	75659671db	Fix a warning from Hibernate (#645 ) Hibernate complained that ContactResource.getTransferData should not be final: WARN: HHH000305: Could not create proxy factory for:google.registry.model.contact.ContactResource org.hibernate.HibernateException: Getter methods of lazy classes cannot be final: google.registry.model.contact.ContactResource#getTransferData	2020-06-24 10:02:42 -04:00
Michael MullerandGitHub	c9d4ffb233	Convert all VKeys to symmetric VKeys (#641 ) * Convert hosts to symmetric VKey * Convert ContactResource to symmetric VKeys * Convert BillingEvents to symmetric VKeys * Converted PollMessage to symmetric VKeys * Convert AllocationToken to symmetric VKeys * Remove static methods, get everything working * Changes requested in review. * Removed newly introduced createOfy() calls	2020-06-24 08:02:11 -04:00
Weimin YuandGitHub	11ec4d64f2	Set up JpaTransactionManager in BEAM pipelines (#639 ) * Set up JpaTransactionManager in BEAM pipelines Added modules and utilities to create JpaTransactionManager in BEAM pipelines. Not wanting to set up AppEngine Remote API to access Keyring in the Datastore, we instead use the credential files in GCS, which are used by Spinnaker/Cloud Build and desktop access. Added utility to download, decrypt, and parse the file. Also added/modified dagger modules.	2020-06-23 11:04:52 -04:00