Refactor pipline for Datastore backup loading (#628)

* Refactor pipline for Datastore backup loading

Refactored pipeline transforms.

Added testing utilities that handles assertions better.

Investigated and documented challenges in serializing Ofy entities
without side effects.

README.md

@@ -33,7 +33,7 @@ running system:
 *   View the source code for the [GAE app](https://github.com/google/nomulus/tree/master/core/src/main/java/google/registry)
     and for the [GKE proxy](https://github.com/google/nomulus/tree/master/proxy/src/main/java/google/registry)
 *   [Other docs](https://github.com/google/nomulus/tree/master/docs)
-*   [Javadoc](https://nomulus.foo/javadoc/latest/)
+*   [Javadoc](https://javadoc.nomulus.foo/)
 *   [Nomulus discussion
     group](https://groups.google.com/forum/#!forum/nomulus-discuss), for any
     other questions

build.gradle

@@ -197,6 +197,10 @@ task runPresubmits(type: Exec) {
   args('config/presubmits.py')
 }
 
+def javadocSource = []
+def javadocClasspath = []
+def javadocDependentTasks = []
+
 subprojects {
   // Skip no-op project
   if (project.name == 'services') return
@@ -317,6 +321,10 @@ subprojects {
       }
     }
   }
+
+  javadocSource << project.sourceSets.main.allJava
+  javadocClasspath << project.sourceSets.main.compileClasspath
+  javadocDependentTasks << project.tasks.compileJava
 }
 
 // If "-P verboseTestOutput=true" is passed in, configure all subprojects to dump all of their
@@ -445,3 +453,19 @@ task javaIncrementalFormatApply {
 }
 
 tasks.build.dependsOn(tasks.javaIncrementalFormatCheck)
+
+task javadoc(type: Javadoc) {
+  source javadocSource
+  classpath = files(javadocClasspath)
+  destinationDir = file("${buildDir}/docs/javadoc")
+  options.encoding = "UTF-8"
+  // In a lot of places we don't write @return so suppress warnings about that.
+  options.addBooleanOption('Xdoclint:all,-missing', true)
+  options.addBooleanOption("-allow-script-in-comments",true)
+  options.tags = ["type:a:Generic Type",
+                  "error:a:Expected Error"]
+}
+
+tasks.build.dependsOn(tasks.javadoc)
+
+javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }

common/src/main/java/google/registry/util/Sleeper.java

@@ -21,7 +21,6 @@ import org.joda.time.ReadableDuration;
  * An object which accepts requests to put the current thread to sleep.
  *
  * @see SystemSleeper
- * @see google.registry.testing.FakeSleeper
  */
 @ThreadSafe
 public interface Sleeper {

config/presubmits.py

@@ -99,6 +99,15 @@ PRESUBMITS = {
         "System.(out|err).println is only allowed in tools/ packages. Please "
         "use a logger instead.",
 
+    # ObjectifyService.register is restricted to main/ or AppEngineRule.
+    PresubmitCheck(
+        r".*\bObjectifyService\.register", "java", {
+            "/build/", "/generated/", "node_modules/", "src/main/",
+            "AppEngineRule.java"
+        }):
+      "ObjectifyService.register is not allowed in tests. Please use "
+      "AppengineRule.register instead.",
+
     # PostgreSQLContainer instantiation must specify docker tag
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",

core/build.gradle

@@ -295,6 +295,8 @@ dependencies {
   testAnnotationProcessor deps['com.google.auto.value:auto-value']
   annotationProcessor deps['com.google.dagger:dagger-compiler']
   testAnnotationProcessor deps['com.google.dagger:dagger-compiler']
+  annotationProcessor project(':processor')
+  testAnnotationProcessor project(':processor')
 
   testCompile deps['com.google.appengine:appengine-testing']
   testCompile deps['com.google.guava:guava-testlib']
@@ -707,7 +709,10 @@ task outcastTest(type: FilteringTest) {
   tests = outcastTestPatterns
 
   // Sets the maximum number of test executors that may exist at the same time.
-  maxParallelForks 5
+  // Note that this number appears to contribute to NoClassDefFoundError
+  // exceptions on certain machines and distros. The root cause is unclear.
+  // Try reducing this number if you experience similar problems.
+  maxParallelForks 3
 }
 
 // Whitebox test verifying that RegistryTool can be instantiated. Note the
@@ -875,6 +880,9 @@ task standardTest(type: FilteringTest) {
   // forkEvery 1
 
   // Sets the maximum number of test executors that may exist at the same time.
+  // Also, Gradle executes tests in 1 thread and some of our test infrastructures
+  // depend on that, e.g. DualDatabaseTestInvocationContextProvider injects
+  // different implementation of TransactionManager into TransactionManagerFactory.
   maxParallelForks 5
 
   systemProperty 'test.projectRoot', rootProject.projectRootDir

core/gradle/dependency-locks/annotationProcessor.lockfile

@@ -12,24 +12,24 @@ com.google.dagger:dagger-producers:2.21
 com.google.dagger:dagger-spi:2.21
 com.google.dagger:dagger:2.21
 com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
+com.google.errorprone:error_prone_annotations:2.3.4
 com.google.errorprone:error_prone_check_api:2.3.3
 com.google.errorprone:error_prone_core:2.3.3
 com.google.errorprone:error_prone_type_annotations:2.3.3
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:27.0.1-jre
+com.google.guava:guava:28.2-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.j2objc:j2objc-annotations:1.1
+com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-com.squareup:javapoet:1.11.1
+com.squareup:javapoet:1.12.1
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
+javax.persistence:javax.persistence-api:2.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:2.5.3
+org.checkerframework:checker-qual:2.10.0
 org.checkerframework:dataflow:2.5.3
 org.checkerframework:javacutil:2.5.3
-org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2

core/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -12,24 +12,24 @@ com.google.dagger:dagger-producers:2.21
 com.google.dagger:dagger-spi:2.21
 com.google.dagger:dagger:2.21
 com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
+com.google.errorprone:error_prone_annotations:2.3.4
 com.google.errorprone:error_prone_check_api:2.3.3
 com.google.errorprone:error_prone_core:2.3.3
 com.google.errorprone:error_prone_type_annotations:2.3.3
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:27.0.1-jre
+com.google.guava:guava:28.2-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.j2objc:j2objc-annotations:1.1
+com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-com.squareup:javapoet:1.11.1
+com.squareup:javapoet:1.12.1
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
+javax.persistence:javax.persistence-api:2.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:2.5.3
+org.checkerframework:checker-qual:2.10.0
 org.checkerframework:dataflow:2.5.3
 org.checkerframework:javacutil:2.5.3
-org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2

core/src/main/java/google/registry/backup/AppEngineEnvironment.java

@@ -0,0 +1,66 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.apphosting.api.ApiProxy;
+import com.google.apphosting.api.ApiProxy.Environment;
+import com.google.common.collect.ImmutableMap;
+import java.io.Closeable;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+
+/**
+ * Sets up a placeholder {@link Environment} on a non-AppEngine platform so that Datastore Entities
+ * can be deserialized. See {@code DatastoreEntityExtension} in test source for more information.
+ */
+public class AppEngineEnvironment implements Closeable {
+
+  private static final Environment PLACEHOLDER_ENV = createAppEngineEnvironment();
+
+  private boolean isPlaceHolderNeeded;
+
+  public AppEngineEnvironment() {
+    isPlaceHolderNeeded = ApiProxy.getCurrentEnvironment() == null;
+    // isPlaceHolderNeeded may be true when we are invoked in a test with AppEngineRule.
+    if (isPlaceHolderNeeded) {
+      ApiProxy.setEnvironmentForCurrentThread(PLACEHOLDER_ENV);
+    }
+  }
+
+  @Override
+  public void close() {
+    if (isPlaceHolderNeeded) {
+      ApiProxy.setEnvironmentForCurrentThread(null);
+    }
+  }
+
+  /** Returns a placeholder {@link Environment} that can return hardcoded AppId and Attributes. */
+  private static Environment createAppEngineEnvironment() {
+    return (Environment)
+        Proxy.newProxyInstance(
+            Environment.class.getClassLoader(),
+            new Class[] {Environment.class},
+            (Object proxy, Method method, Object[] args) -> {
+              switch (method.getName()) {
+                case "getAppId":
+                  return "PlaceholderAppId";
+                case "getAttributes":
+                  return ImmutableMap.<String, Object>of();
+                default:
+                  throw new UnsupportedOperationException(method.getName());
+              }
+            });
+  }
+}

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -0,0 +1,95 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.backup.BackupUtils.createDeserializingIterator;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Streams;
+import google.registry.model.ImmutableObject;
+import google.registry.model.ofy.CommitLogCheckpoint;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Iterator;
+import java.util.stream.Stream;
+
+/**
+ * Helpers for reading CommitLog records from a file.
+ *
+ * <p>This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial
+ * population of the Cloud SQL database.
+ */
+public final class CommitLogImports {
+
+  private CommitLogImports() {}
+
+  /**
+   * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link
+   * ImmutableList} of {@link VersionedEntity} records. Upon completion the {@code inputStream} is
+   * closed.
+   *
+   * <p>The returned list may be empty, since CommitLogs are written at fixed intervals regardless
+   * if actual changes exist.
+   *
+   * <p>A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated)
+   * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence
+   * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
+   * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
+   */
+  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
+        InputStream input = new BufferedInputStream(inputStream)) {
+      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
+      checkState(commitLogs.hasNext());
+      checkState(commitLogs.next() instanceof CommitLogCheckpoint);
+
+      return Streams.stream(commitLogs)
+          .map(
+              e ->
+                  e instanceof CommitLogManifest
+                      ? VersionedEntity.fromManifest((CommitLogManifest) e)
+                      : Stream.of(VersionedEntity.fromMutation((CommitLogMutation) e)))
+          .flatMap(s -> s)
+          .collect(ImmutableList.toImmutableList());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /** Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link File}. */
+  public static ImmutableList<VersionedEntity> loadEntities(File commitLogFile) {
+    try {
+      return loadEntities(new FileInputStream(commitLogFile));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link
+   * ReadableByteChannel}.
+   */
+  public static ImmutableList<VersionedEntity> loadEntities(ReadableByteChannel channel) {
+    return loadEntities(Channels.newInputStream(channel));
+  }
+}

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -0,0 +1,198 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.appengine.api.datastore.Key;
+import com.google.auto.value.AutoValue;
+import com.google.auto.value.extension.memoized.Memoized;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+
+/**
+ * A Datastore {@link Entity Entity's} serialized state with timestamp. The intended use case is a
+ * multi-stage pipeline where an Entity's Java form is not needed in most stages.
+ *
+ * <p>For a new or updated Entity, its serialized bytes are stored along with its Datastore {@link
+ * Key}. For a deleted entity, only its Datastore {@link Key} is stored, and the {@link
+ * #entityProtoBytes} field is left unset.
+ *
+ * <p>Storing raw bytes is motivated by two factors. First, since I/O is frequent and the Java
+ * objects are rarely needed in our target use case, storing raw bytes is the most efficient
+ * approach. More importantly, due to our data model and our customization of {@link
+ * google.registry.model.ofy.ObjectifyService ObjectifyService}, it is challenging to implement a
+ * serializer for Objectify entities that preserves the value of all properties. Without such
+ * serializers, Objectify entities cannot be used in a pipeline.
+ *
+ * <p>Objectify entities do not implement {@link Serializable}, serialization of such objects is as
+ * follows:
+ *
+ * <ul>
+ *   <li>Convert an Objectify entity to a Datastore {@link Entity}: {@code
+ *       ofy().save().toEntity(..)}
+ *   <li>Entity is serializable, but the more efficient approach is to convert an Entity to a
+ *       ProtocolBuffer ({@link com.google.storage.onestore.v3.OnestoreEntity.EntityProto}) and then
+ *       to raw bytes.
+ * </ul>
+ *
+ * <p>When the first conversion above is applied to an Objectify entity, a property value in the
+ * output may differ from the input in two situations:
+ *
+ * <ul>
+ *   <li>If a property is of an assign-on-persist data type, e.g., {@link
+ *       google.registry.model.UpdateAutoTimestamp}.
+ *   <li>If it is related to CommitLog management, e.g., {@link google.registry.model.EppResource
+ *       EppResource.revisions}.
+ * </ul>
+ *
+ * <p>Working around the side effects caused by our customization is difficult. Any solution would
+ * likely rely on Objectify's stack of context. However, many Objectify invocations in our code base
+ * are hardcoded to call the customized version of ObjectifyService, rendering Objectify's stack
+ * useless.
+ *
+ * <p>For now, this inability to use Objectify entities in pipelines is mostly a testing problem: we
+ * can not perform {@link org.apache.beam.sdk.testing.PAssert BEAM pipeline assertions} on Objectify
+ * entities. {@code InitSqlTestUtils.assertContainsExactlyElementsIn} is an example of a workaround.
+ *
+ * <p>Note that {@link Optional java.util.Optional} is not serializable, therefore cannot be used as
+ * property type in this class.
+ */
+@AutoValue
+public abstract class VersionedEntity implements Serializable {
+
+  private static final long serialVersionUID = 1L;
+
+  public abstract long commitTimeMills();
+
+  /** The {@link Key} of the {@link Entity}. */
+  public abstract Key key();
+
+  /** Serialized form of the {@link Entity}. This property is {@code null} for a deleted Entity. */
+  @Nullable
+  abstract ImmutableBytes entityProtoBytes();
+
+  @Memoized
+  public Optional<Entity> getEntity() {
+    return Optional.ofNullable(entityProtoBytes())
+        .map(ImmutableBytes::getBytes)
+        .map(EntityTranslator::createFromPbBytes);
+  }
+
+  public boolean isDelete() {
+    return entityProtoBytes() == null;
+  }
+
+  /**
+   * Converts deleted entity keys in {@code manifest} into a {@link Stream} of {@link
+   * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
+   * information.
+   */
+  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+    long commitTimeMillis = manifest.getCommitTime().getMillis();
+    return manifest.getDeletions().stream()
+        .map(com.googlecode.objectify.Key::getRaw)
+        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+  }
+
+  /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
+  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+    return from(
+        com.googlecode.objectify.Key.create(mutation).getParent().getId(),
+        mutation.getEntityProtoBytes());
+  }
+
+  public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
+    return builder()
+        .entityProtoBytes(entityProtoBytes)
+        .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
+        .commitTimeMills(commitTimeMillis)
+        .build();
+  }
+
+  static Builder builder() {
+    return new AutoValue_VersionedEntity.Builder();
+  }
+
+  @AutoValue.Builder
+  public abstract static class Builder {
+
+    public abstract Builder commitTimeMills(long commitTimeMillis);
+
+    abstract Builder entityProtoBytes(ImmutableBytes bytes);
+
+    public abstract Builder key(Key key);
+
+    public abstract VersionedEntity build();
+
+    public Builder entityProtoBytes(byte[] bytes) {
+      return entityProtoBytes(new ImmutableBytes(bytes));
+    }
+  }
+
+  /**
+   * Wraps a byte array and prevents it from being modified by its original owner.
+   *
+   * <p>While this class seems an overkill, it exists for two reasons:
+   *
+   * <ul>
+   *   <li>It is easier to override the {@link #equals} method here (for value-equivalence check)
+   *       than to override the AutoValue-generated {@code equals} method.
+   *   <li>To appease the style checker, which forbids arrays as AutoValue property.
+   * </ul>
+   */
+  static final class ImmutableBytes implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private final byte[] bytes;
+
+    ImmutableBytes(byte[] bytes) {
+      this.bytes = Arrays.copyOf(bytes, bytes.length);
+    }
+
+    /**
+     * Returns the saved byte array. Invocation is restricted to trusted callers, who must not
+     * modify the array.
+     */
+    byte[] getBytes() {
+      return bytes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof ImmutableBytes)) {
+        return false;
+      }
+      ImmutableBytes that = (ImmutableBytes) o;
+      // Do not use Objects.equals, which checks reference identity instead of data in array.
+      return Arrays.equals(bytes, that.bytes);
+    }
+
+    @Override
+    public int hashCode() {
+      // Do not use Objects.hashCode, which hashes the reference, not the data in array.
+      return Arrays.hashCode(bytes);
+    }
+  }
+}

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -150,7 +150,7 @@ public final class AsyncTaskEnqueuer {
 
   /** Enqueues a task to asynchronously refresh DNS for a renamed host. */
   public void enqueueAsyncDnsRefresh(HostResource host, DateTime now) {
-    VKey<HostResource> hostKey = host.createKey();
+    VKey<HostResource> hostKey = host.createVKey();
     logger.atInfo().log("Enqueuing async DNS refresh for renamed host %s.", hostKey);
     addTaskToQueueWithRetry(
         asyncDnsRefreshPullQueue,

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -283,7 +283,9 @@ public class DeleteContactsAndHostsAction implements Runnable {
     /** Determine whether the target resource is a linked resource on the domain. */
     private boolean isLinked(DomainBase domain, Key<? extends EppResource> resourceKey) {
       if (resourceKey.getKind().equals(KIND_CONTACT)) {
-        return domain.getReferencedContacts().contains(resourceKey);
+        return domain
+            .getReferencedContacts()
+            .contains(VKey.createOfy(ContactResource.class, (Key<ContactResource>) resourceKey));
       } else if (resourceKey.getKind().equals(KIND_HOST)) {
         return domain
             .getNameservers()
@@ -345,7 +347,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
       String resourceClientId = resource.getPersistedCurrentSponsorClientId();
       if (resource instanceof HostResource && ((HostResource) resource).isSubordinate()) {
         resourceClientId =
-            ofy().load().key(((HostResource) resource).getSuperordinateDomain()).now()
+            tm().load(((HostResource) resource).getSuperordinateDomain())
                 .cloneProjectedAtTime(now)
                 .getCurrentSponsorClientId();
       }
@@ -424,7 +426,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
       if (resource instanceof HostResource) {
         return ImmutableList.of(
             HostPendingActionNotificationResponse.create(
-                ((HostResource) resource).getFullyQualifiedHostName(), deleteAllowed, trid, now));
+                ((HostResource) resource).getHostName(), deleteAllowed, trid, now));
       } else if (resource instanceof ContactResource) {
         return ImmutableList.of(
             ContactPendingActionNotificationResponse.create(
@@ -463,11 +465,12 @@ public class DeleteContactsAndHostsAction implements Runnable {
       } else if (existingResource instanceof HostResource) {
         HostResource host = (HostResource) existingResource;
         if (host.isSubordinate()) {
-          dnsQueue.addHostRefreshTask(host.getFullyQualifiedHostName());
-          ofy().save().entity(
-              ofy().load().key(host.getSuperordinateDomain()).now().asBuilder()
-                  .removeSubordinateHost(host.getFullyQualifiedHostName())
-                  .build());
+          dnsQueue.addHostRefreshTask(host.getHostName());
+          tm().saveNewOrUpdate(
+                  tm().load(host.getSuperordinateDomain())
+                      .asBuilder()
+                      .removeSubordinateHost(host.getHostName())
+                      .build());
         }
       } else {
         throw new IllegalStateException(

core/src/main/java/google/registry/batch/DeleteProberDataAction.java

@@ -177,7 +177,7 @@ public class DeleteProberDataAction implements Runnable {
         return;
       }
 
-      String domainName = domain.getFullyQualifiedDomainName();
+      String domainName = domain.getDomainName();
       if (domainName.equals("nic." + domain.getTld())) {
         getContext().incrementCounter("skipped, NIC domain");
         return;
@@ -265,7 +265,7 @@ public class DeleteProberDataAction implements Runnable {
           // mapreduce runs anyway.
           ofy().save().entities(deletedDomain, historyEntry);
           updateForeignKeyIndexDeletionTime(deletedDomain);
-          dnsQueue.addDomainRefreshTask(deletedDomain.getFullyQualifiedDomainName());
+          dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
         }
       );
     }

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -155,89 +155,100 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       }
       int numBillingEventsSaved = 0;
       try {
-        numBillingEventsSaved = tm().transactNew(() -> {
-          ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder =
-              new ImmutableSet.Builder<>();
-          final Registry tld = Registry.get(getTldFromDomainName(recurring.getTargetId()));
+        numBillingEventsSaved =
+            tm().transactNew(
+                    () -> {
+                      ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder =
+                          new ImmutableSet.Builder<>();
+                      final Registry tld =
+                          Registry.get(getTldFromDomainName(recurring.getTargetId()));
 
-          // Determine the complete set of times at which this recurring event should occur
-          // (up to and including the runtime of the mapreduce).
-          Iterable<DateTime> eventTimes =
-              recurring.getRecurrenceTimeOfYear().getInstancesInRange(Range.closed(
-                  recurring.getEventTime(),
-                  earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
+                      // Determine the complete set of times at which this recurring event should
+                      // occur (up to and including the runtime of the mapreduce).
+                      Iterable<DateTime> eventTimes =
+                          recurring
+                              .getRecurrenceTimeOfYear()
+                              .getInstancesInRange(
+                                  Range.closed(
+                                      recurring.getEventTime(),
+                                      earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
 
-          // Convert these event times to billing times
-          final ImmutableSet<DateTime> billingTimes =
-              getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
+                      // Convert these event times to billing times
+                      final ImmutableSet<DateTime> billingTimes =
+                          getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
 
-          Key<? extends EppResource> domainKey = recurring.getParentKey().getParent();
-          Iterable<OneTime> oneTimesForDomain =
-              ofy().load().type(OneTime.class).ancestor(domainKey);
+                      Key<? extends EppResource> domainKey = recurring.getParentKey().getParent();
+                      Iterable<OneTime> oneTimesForDomain =
+                          ofy().load().type(OneTime.class).ancestor(domainKey);
 
-          // Determine the billing times that already have OneTime events persisted.
-          ImmutableSet<DateTime> existingBillingTimes =
-              getExistingBillingTimes(oneTimesForDomain, recurring);
+                      // Determine the billing times that already have OneTime events persisted.
+                      ImmutableSet<DateTime> existingBillingTimes =
+                          getExistingBillingTimes(oneTimesForDomain, recurring);
 
-          ImmutableSet.Builder<HistoryEntry> historyEntriesBuilder =
-              new ImmutableSet.Builder<>();
-          // Create synthetic OneTime events for all billing times that do not yet have an event
-          // persisted.
-          for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
-            // Construct a new HistoryEntry that parents over the OneTime
-            HistoryEntry historyEntry = new HistoryEntry.Builder()
-                .setBySuperuser(false)
-                .setClientId(recurring.getClientId())
-                .setModificationTime(tm().getTransactionTime())
-                .setParent(domainKey)
-                .setPeriod(Period.create(1, YEARS))
-                .setReason("Domain autorenewal by ExpandRecurringBillingEventsAction")
-                .setRequestedByRegistrar(false)
-                .setType(DOMAIN_AUTORENEW)
-                // Don't write a domain transaction record if the recurrence was ended prior to the
-                // billing time (i.e. a domain was deleted during the autorenew grace period).
-                .setDomainTransactionRecords(
-                    recurring.getRecurrenceEndTime().isBefore(billingTime)
-                        ? ImmutableSet.of()
-                        : ImmutableSet.of(
-                            DomainTransactionRecord.create(
-                                tld.getTldStr(),
-                                // We report this when the autorenew grace period ends
-                                billingTime,
-                                TransactionReportField.netRenewsFieldFromYears(1),
-                                1)))
-                .build();
-            historyEntriesBuilder.add(historyEntry);
+                      ImmutableSet.Builder<HistoryEntry> historyEntriesBuilder =
+                          new ImmutableSet.Builder<>();
+                      // Create synthetic OneTime events for all billing times that do not yet have
+                      // an event persisted.
+                      for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
+                        // Construct a new HistoryEntry that parents over the OneTime
+                        HistoryEntry historyEntry =
+                            new HistoryEntry.Builder()
+                                .setBySuperuser(false)
+                                .setClientId(recurring.getClientId())
+                                .setModificationTime(tm().getTransactionTime())
+                                .setParent(domainKey)
+                                .setPeriod(Period.create(1, YEARS))
+                                .setReason(
+                                    "Domain autorenewal by ExpandRecurringBillingEventsAction")
+                                .setRequestedByRegistrar(false)
+                                .setType(DOMAIN_AUTORENEW)
+                                // Don't write a domain transaction record if the recurrence was
+                                // ended prior to the billing time (i.e. a domain was deleted
+                                // during the autorenew grace period).
+                                .setDomainTransactionRecords(
+                                    recurring.getRecurrenceEndTime().isBefore(billingTime)
+                                        ? ImmutableSet.of()
+                                        : ImmutableSet.of(
+                                            DomainTransactionRecord.create(
+                                                tld.getTldStr(),
+                                                // We report this when the autorenew grace period
+                                                // ends
+                                                billingTime,
+                                                TransactionReportField.netRenewsFieldFromYears(1),
+                                                1)))
+                                .build();
+                        historyEntriesBuilder.add(historyEntry);
 
-            DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
-            // Determine the cost for a one-year renewal.
-            Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
-            syntheticOneTimesBuilder.add(new OneTime.Builder()
-                .setBillingTime(billingTime)
-                .setClientId(recurring.getClientId())
-                .setCost(renewCost)
-                .setEventTime(eventTime)
-                .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
-                .setParent(historyEntry)
-                .setPeriodYears(1)
-                .setReason(recurring.getReason())
-                .setSyntheticCreationTime(executeTime)
-                .setCancellationMatchingBillingEvent(Key.create(recurring))
-                .setTargetId(recurring.getTargetId())
-                .build());
-          }
-          Set<HistoryEntry> historyEntries = historyEntriesBuilder.build();
-          Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
-          if (!isDryRun) {
-            ImmutableSet<ImmutableObject> entitiesToSave =
-                new ImmutableSet.Builder<ImmutableObject>()
-                    .addAll(historyEntries)
-                    .addAll(syntheticOneTimes)
-                    .build();
-            ofy().save().entities(entitiesToSave).now();
-          }
-          return syntheticOneTimes.size();
-        });
+                        DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
+                        // Determine the cost for a one-year renewal.
+                        Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
+                        syntheticOneTimesBuilder.add(
+                            new OneTime.Builder()
+                                .setBillingTime(billingTime)
+                                .setClientId(recurring.getClientId())
+                                .setCost(renewCost)
+                                .setEventTime(eventTime)
+                                .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
+                                .setParent(historyEntry)
+                                .setPeriodYears(1)
+                                .setReason(recurring.getReason())
+                                .setSyntheticCreationTime(executeTime)
+                                .setCancellationMatchingBillingEvent(recurring.createVKey())
+                                .setTargetId(recurring.getTargetId())
+                                .build());
+                      }
+                      Set<HistoryEntry> historyEntries = historyEntriesBuilder.build();
+                      Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
+                      if (!isDryRun) {
+                        ImmutableSet<ImmutableObject> entitiesToSave =
+                            new ImmutableSet.Builder<ImmutableObject>()
+                                .addAll(historyEntries)
+                                .addAll(syntheticOneTimes)
+                                .build();
+                        ofy().save().entities(entitiesToSave).now();
+                      }
+                      return syntheticOneTimes.size();
+                    });
       } catch (Throwable t) {
         getContext().incrementCounter("error: " + t.getClass().getSimpleName());
         getContext().incrementCounter(ERROR_COUNTER);
@@ -279,7 +290,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       return Streams.stream(oneTimesForDomain)
           .filter(
               billingEvent ->
-                  Key.create(recurringEvent)
+                  recurringEvent
+                      .createVKey()
                       .equals(billingEvent.getCancellationMatchingBillingEvent()))
           .map(OneTime::getBillingTime)
           .collect(toImmutableSet());

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -216,11 +216,11 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
       }
       if (referencingHostKey != null) {
         retrier.callWithRetry(
-            () -> dnsQueue.addDomainRefreshTask(domain.getFullyQualifiedDomainName()),
+            () -> dnsQueue.addDomainRefreshTask(domain.getDomainName()),
             TransientFailureException.class);
         logger.atInfo().log(
             "Enqueued DNS refresh for domain %s referenced by host %s.",
-            domain.getFullyQualifiedDomainName(), referencingHostKey);
+            domain.getDomainName(), referencingHostKey);
         getContext().incrementCounter("domains refreshed");
       } else {
         getContext().incrementCounter("domains not refreshed");

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -96,7 +96,7 @@ public class RelockDomainAction implements Runnable {
         String message =
             String.format(
                 "Domain %s is already manually relocked, skipping automated relock.",
-                domain.getFullyQualifiedDomainName());
+                domain.getDomainName());
         logger.atInfo().log(message);
         // SC_NO_CONTENT (204) skips retry -- see the comment below
         response.setStatus(SC_NO_CONTENT);
@@ -144,7 +144,7 @@ public class RelockDomainAction implements Runnable {
 
   private void verifyDomainAndLockState(RegistryLock oldLock, DomainBase domain) {
     // Domain shouldn't be deleted or have a pending transfer/delete
-    String domainName = domain.getFullyQualifiedDomainName();
+    String domainName = domain.getDomainName();
     checkArgument(
         !DateTimeUtils.isAtOrAfter(jpaTm().getTransactionTime(), domain.getDeletionTime()),
         "Domain %s has been deleted",

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -0,0 +1,93 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Streams;
+import org.joda.time.DateTime;
+
+/**
+ * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a
+ * Datastore export and Nomulus CommitLogs that overlap with the export.
+ */
+public final class BackupPaths {
+
+  private BackupPaths() {}
+
+  private static final String WILDCARD_CHAR = "*";
+  private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/input-%s";
+
+  public static final String COMMIT_LOG_NAME_PREFIX = "commit_diff_until_";
+  private static final String COMMIT_LOG_PATTERN_TEMPLATE = "%s/" + COMMIT_LOG_NAME_PREFIX + "*";
+
+  /**
+   * Returns a regex pattern that matches all Datastore export files of a given {@code kind}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kind the 'kind' of the Datastore entity
+   */
+  public static String getExportFileNamePattern(String exportDir, String kind) {
+    checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
+    checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, WILDCARD_CHAR);
+  }
+
+  /**
+   * Returns an {@link ImmutableList} of regex patterns that match all Datastore export files of the
+   * given {@code kinds}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kinds all entity 'kinds' to be matched
+   */
+  public static ImmutableList<String> getExportFilePatterns(
+      String exportDir, Iterable<String> kinds) {
+    checkNotNull(kinds, "kinds");
+    return Streams.stream(kinds)
+        .map(kind -> getExportFileNamePattern(exportDir, kind))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  /**
+   * Returns the fully qualified path of a Datastore export file with the given {@code kind} and
+   * {@code shard}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kind the 'kind' of the Datastore entity
+   * @param shard an integer suffix of the file name
+   */
+  public static String getExportFileNameByShard(String exportDir, String kind, int shard) {
+    checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
+    checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
+    checkArgument(shard >= 0, "Negative shard %s not allowed.", shard);
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, Integer.toString(shard));
+  }
+
+  /** Returns an {@link ImmutableList} of regex patterns that match all CommitLog files. */
+  public static ImmutableList<String> getCommitLogFilePatterns(String commitLogDir) {
+    return ImmutableList.of(String.format(COMMIT_LOG_PATTERN_TEMPLATE, commitLogDir));
+  }
+
+  /** Gets the Commit timestamp from a CommitLog file name. */
+  public static DateTime getCommitLogTimestamp(String fileName) {
+    checkArgument(!isNullOrEmpty(fileName), "Null or empty fileName.");
+    int start = fileName.lastIndexOf(COMMIT_LOG_NAME_PREFIX);
+    checkArgument(start >= 0, "Illegal file name %s.", fileName);
+    return DateTime.parse(fileName.substring(start + COMMIT_LOG_NAME_PREFIX.length()));
+  }
+}

core/src/main/java/google/registry/beam/initsql/README.md

@@ -0,0 +1,3 @@
+## Summary
+
+This package contains a BEAM pipeline that populates a Cloud SQL database from a Datastore backup.

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -0,0 +1,195 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
+import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+
+import avro.shaded.com.google.common.collect.Iterators;
+import com.google.common.annotations.VisibleForTesting;
+import google.registry.backup.CommitLogImports;
+import google.registry.backup.VersionedEntity;
+import google.registry.tools.LevelDbLogReader;
+import java.util.Collection;
+import java.util.Iterator;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.ProcessFunction;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.DateTime;
+
+/**
+ * {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export
+ * files and Nomulus CommitLog files.
+ */
+public final class Transforms {
+
+  private Transforms() {}
+
+  /**
+   * The commitTimestamp assigned to all entities loaded from a Datastore export file. The exact
+   * value does not matter, but it must be lower than the timestamps of real CommitLog records.
+   */
+  @VisibleForTesting static final long EXPORT_ENTITY_TIME_STAMP = START_OF_TIME.getMillis();
+
+  /**
+   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
+   * all Datastore CommitLog files.
+   */
+  public static PTransform<PBegin, PCollection<String>> getCommitLogFilePatterns(
+      String commitLogDir) {
+    return toStringPCollection(BackupPaths.getCommitLogFilePatterns(commitLogDir));
+  }
+
+  /**
+   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
+   * all Datastore export files of the given {@code kinds}.
+   */
+  public static PTransform<PBegin, PCollection<String>> getDatastoreExportFilePatterns(
+      String exportDir, Collection<String> kinds) {
+    return toStringPCollection(getExportFilePatterns(exportDir, kinds));
+  }
+
+  /**
+   * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}.
+   */
+  public static PTransform<PCollection<String>, PCollection<Metadata>> getFilesByPatterns() {
+    return new PTransform<PCollection<String>, PCollection<Metadata>>() {
+      @Override
+      public PCollection<Metadata> expand(PCollection<String> input) {
+        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+      }
+    };
+  }
+
+  /**
+   * Returns CommitLog files with timestamps between {@code fromTime} (inclusive) and {@code
+   * endTime} (exclusive).
+   */
+  public static PTransform<PCollection<? extends String>, PCollection<String>>
+      filterCommitLogsByTime(DateTime fromTime, DateTime toTime) {
+    return ParDo.of(new FilterCommitLogFileByTime(fromTime, toTime));
+  }
+
+  /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
+  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
+      loadExportDataFromFiles() {
+    return processFiles(
+        new BackupFileReader(
+            file ->
+                Iterators.transform(
+                    LevelDbLogReader.from(file.open()),
+                    (byte[] bytes) -> VersionedEntity.from(EXPORT_ENTITY_TIME_STAMP, bytes))));
+  }
+
+  /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
+  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
+      loadCommitLogsFromFiles() {
+    return processFiles(
+        new BackupFileReader(file -> CommitLogImports.loadEntities(file.open()).iterator()));
+  }
+
+  /**
+   * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
+   * given {@link Iterable}.
+   */
+  static PTransform<PBegin, PCollection<String>> toStringPCollection(Iterable<String> strings) {
+    return Create.of(strings).withCoder(StringUtf8Coder.of());
+  }
+
+  /**
+   * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using
+   * caller-provided {@code transformer}.
+   */
+  static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
+      DoFn<ReadableFile, VersionedEntity> transformer) {
+    return new PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>() {
+      @Override
+      public PCollection<VersionedEntity> expand(PCollection<Metadata> input) {
+        return input
+            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+            .apply(transformer.getClass().getSimpleName(), ParDo.of(transformer));
+        // TODO(weiminyu): reshuffle to enable dynamic work rebalance per beam dev guide
+      }
+    };
+  }
+
+  private static class FilterCommitLogFileByTime extends DoFn<String, String> {
+    private final DateTime fromTime;
+    private final DateTime toTime;
+
+    public FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
+      checkNotNull(fromTime, "fromTime");
+      checkNotNull(toTime, "toTime");
+      checkArgument(
+          fromTime.isBefore(toTime),
+          "Invalid time range: fromTime (%s) is before endTime (%s)",
+          fromTime,
+          toTime);
+      this.fromTime = fromTime;
+      this.toTime = toTime;
+    }
+
+    @ProcessElement
+    public void processElement(@Element String fileName, OutputReceiver<String> out) {
+      DateTime timestamp = getCommitLogTimestamp(fileName);
+      if (isBeforeOrAt(fromTime, timestamp) && timestamp.isBefore(toTime)) {
+        out.output(fileName);
+      }
+    }
+  }
+
+  /**
+   * Reads from a Datastore backup file and converts its content into {@link VersionedEntity
+   * VersionedEntities}.
+   *
+   * <p>The input file may be either a LevelDb file from a Datastore export or a CommitLog file
+   * generated by the Nomulus server. In either case, the file contains variable-length records and
+   * must be read sequentially from the beginning. If the read fails, the file needs to be retried
+   * from the beginning.
+   */
+  private static class BackupFileReader extends DoFn<ReadableFile, VersionedEntity> {
+    private final ProcessFunction<ReadableFile, Iterator<VersionedEntity>> reader;
+
+    private BackupFileReader(ProcessFunction<ReadableFile, Iterator<VersionedEntity>> reader) {
+      this.reader = reader;
+    }
+
+    @ProcessElement
+    public void processElement(@Element ReadableFile file, OutputReceiver<VersionedEntity> out) {
+      try {
+        reader.apply(file).forEachRemaining(out::output);
+      } catch (Exception e) {
+        // Let the pipeline use default retry strategy on the whole file. For GCP Dataflow this
+        // means retrying up to 4 times (may include other files grouped with this one), and failing
+        // the pipeline if no success.
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

core/src/main/java/google/registry/beam/spec11/ThreatMatch.java

@@ -24,22 +24,10 @@ import org.json.JSONObject;
 public abstract class ThreatMatch implements Serializable {
 
   private static final String THREAT_TYPE_FIELD = "threatType";
-  private static final String PLATFORM_TYPE_FIELD = "platformType";
-  private static final String METADATA_FIELD = "threatEntryMetadata";
   private static final String DOMAIN_NAME_FIELD = "fullyQualifiedDomainName";
 
   /** Returns what kind of threat it is (malware, phishing etc.) */
   public abstract String threatType();
-  /** Returns what platforms it affects (Windows, Linux etc.) */
-  abstract String platformType();
-  /**
-   * Returns a String representing a JSON Object containing arbitrary metadata associated with this
-   * threat, or "NONE" if there is no metadata to retrieve.
-   *
-   * <p>This ideally would be a {@link JSONObject} type, but can't be due to serialization
-   * requirements.
-   */
-  abstract String metadata();
   /** Returns the fully qualified domain name [SLD].[TLD] of the matched threat. */
   public abstract String fullyQualifiedDomainName();
 
@@ -52,29 +40,19 @@ public abstract class ThreatMatch implements Serializable {
   static ThreatMatch create(JSONObject threatMatchJSON, String fullyQualifiedDomainName)
       throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatchJSON.getString(THREAT_TYPE_FIELD),
-        threatMatchJSON.getString(PLATFORM_TYPE_FIELD),
-        threatMatchJSON.has(METADATA_FIELD)
-            ? threatMatchJSON.getJSONObject(METADATA_FIELD).toString()
-            : "NONE",
-        fullyQualifiedDomainName);
+        threatMatchJSON.getString(THREAT_TYPE_FIELD), fullyQualifiedDomainName);
   }
 
   /** Returns a {@link JSONObject} representing a subset of this object's data. */
   JSONObject toJSON() throws JSONException {
     return new JSONObject()
         .put(THREAT_TYPE_FIELD, threatType())
-        .put(PLATFORM_TYPE_FIELD, platformType())
-        .put(METADATA_FIELD, metadata())
         .put(DOMAIN_NAME_FIELD, fullyQualifiedDomainName());
   }
 
   /** Parses a {@link JSONObject} and returns an equivalent {@link ThreatMatch}. */
   public static ThreatMatch fromJSON(JSONObject threatMatch) throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatch.getString(THREAT_TYPE_FIELD),
-        threatMatch.getString(PLATFORM_TYPE_FIELD),
-        threatMatch.getString(METADATA_FIELD),
-        threatMatch.getString(DOMAIN_NAME_FIELD));
+        threatMatch.getString(THREAT_TYPE_FIELD), threatMatch.getString(DOMAIN_NAME_FIELD));
   }
 }

core/src/main/java/google/registry/bigquery/BigqueryConnection.java

@@ -442,10 +442,8 @@ public class BigqueryConnection implements AutoCloseable {
    * Returns the result of calling queryToLocalTable, but synchronously to avoid spawning new
    * background threads, which App Engine doesn't support.
    *
-   * @see <a href="https://cloud.google.com/appengine/docs/standard/java/runtime#Threads">
-   *   App Engine Runtime</a>
-   *
-   * <p>Returns the results of the query in an ImmutableTable on success.
+   * @see <a href="https://cloud.google.com/appengine/docs/standard/java/runtime#Threads">App Engine
+   *     Runtime</a>
    */
   public ImmutableTable<Integer, TableFieldSchema, Object> queryToLocalTableSync(String querySql) {
     Job job = new Job()
@@ -576,8 +574,6 @@ public class BigqueryConnection implements AutoCloseable {
 
   /**
    * Launch a job, wait for it to complete, but <i>do not</i> check for errors.
-   *
-   * @throws BigqueryJobFailureException
    */
   public Job runJob(Job job, @Nullable AbstractInputStreamContent data) {
     return checkJob(waitForJob(launchJob(job, data)));

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -1512,11 +1512,21 @@ public final class RegistryConfig {
     return CONFIG_SETTINGS.get().hibernate.hikariConnectionTimeout;
   }
 
+  /** Returns the minimum idle connections for HikariCP. */
+  public static String getHibernateHikariMinimumIdle() {
+    return CONFIG_SETTINGS.get().hibernate.hikariMinimumIdle;
+  }
+
   /** Returns the maximum pool size for HikariCP. */
   public static String getHibernateHikariMaximumPoolSize() {
     return CONFIG_SETTINGS.get().hibernate.hikariMaximumPoolSize;
   }
 
+  /** Returns the idle timeout for HikariCP. */
+  public static String getHibernateHikariIdleTimeout() {
+    return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
+  }
+
   /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
   public static String getContactAndHostRoidSuffix() {
     return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -112,7 +112,9 @@ public class RegistryConfigSettings {
     public String connectionIsolation;
     public String logSqlQueries;
     public String hikariConnectionTimeout;
+    public String hikariMinimumIdle;
     public String hikariMaximumPoolSize;
+    public String hikariIdleTimeout;
   }
 
   /** Configuration for Cloud SQL. */

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -207,17 +207,21 @@ hibernate:
 
   # Connection pool configurations.
   hikariConnectionTimeout: 20000
-
   # We occasionally received "Connection is not available, request timed out"
   # exception when setting minimumIdle to 0 and it turned out it is a bug (See
-  # https://github.com/brettwooldridge/HikariCP/issues/1212) in HikariCP and
-  # the workaround is to use a fixed size connection pool.
+  # https://github.com/brettwooldridge/HikariCP/issues/1212) in HikariCP.
+  #
+  # We tried to use a fixed size pool but ran into an issue(See b/155383029),
+  # so we need further investigation to figure out the proper size of the pool.
   #
   # HikariCP also recommends not setting minimumIdle for maximum performance
   # and responsiveness to spike demands (See
   # https://github.com/brettwooldridge/HikariCP).
-  # TODO(b/154720215): Experiment with a smaller pool size.
-  hikariMaximumPoolSize: 20
+  #
+  # TODO(b/154720215): Investigate the long term fix.
+  hikariMinimumIdle: 1
+  hikariMaximumPoolSize: 10
+  hikariIdleTimeout: 300000
 
 cloudSql:
   # jdbc url for the Cloud SQL database.

core/src/main/java/google/registry/dns/DnsQueue.java

@@ -119,35 +119,31 @@ public class DnsQueue {
             .param(PARAM_TLD, tld));
   }
 
-  /**
-   * Adds a task to the queue to refresh the DNS information for the specified subordinate host.
-   */
-  public TaskHandle addHostRefreshTask(String fullyQualifiedHostName) {
-    Optional<InternetDomainName> tld =
-        Registries.findTldForName(InternetDomainName.from(fullyQualifiedHostName));
-    checkArgument(tld.isPresent(),
-        String.format("%s is not a subordinate host to a known tld", fullyQualifiedHostName));
-    return addToQueue(TargetType.HOST, fullyQualifiedHostName, tld.get().toString(), Duration.ZERO);
+  /** Adds a task to the queue to refresh the DNS information for the specified subordinate host. */
+  public TaskHandle addHostRefreshTask(String hostName) {
+    Optional<InternetDomainName> tld = Registries.findTldForName(InternetDomainName.from(hostName));
+    checkArgument(
+        tld.isPresent(), String.format("%s is not a subordinate host to a known tld", hostName));
+    return addToQueue(TargetType.HOST, hostName, tld.get().toString(), Duration.ZERO);
   }
 
   /** Enqueues a task to refresh DNS for the specified domain now. */
-  public TaskHandle addDomainRefreshTask(String fullyQualifiedDomainName) {
-    return addDomainRefreshTask(fullyQualifiedDomainName, Duration.ZERO);
+  public TaskHandle addDomainRefreshTask(String domainName) {
+    return addDomainRefreshTask(domainName, Duration.ZERO);
   }
 
   /** Enqueues a task to refresh DNS for the specified domain at some point in the future. */
-  public TaskHandle addDomainRefreshTask(String fullyQualifiedDomainName, Duration countdown) {
+  public TaskHandle addDomainRefreshTask(String domainName, Duration countdown) {
     return addToQueue(
         TargetType.DOMAIN,
-        fullyQualifiedDomainName,
-        assertTldExists(getTldFromDomainName(fullyQualifiedDomainName)),
+        domainName,
+        assertTldExists(getTldFromDomainName(domainName)),
         countdown);
   }
 
   /** Adds a task to the queue to refresh the DNS information for the specified zone. */
-  public TaskHandle addZoneRefreshTask(String fullyQualifiedZoneName) {
-    return addToQueue(
-        TargetType.ZONE, fullyQualifiedZoneName, fullyQualifiedZoneName, Duration.ZERO);
+  public TaskHandle addZoneRefreshTask(String zoneName) {
+    return addToQueue(TargetType.ZONE, zoneName, zoneName, Duration.ZERO);
   }
 
   /**

core/src/main/java/google/registry/dns/RefreshDnsAction.java

@@ -89,7 +89,7 @@ public final class RefreshDnsAction implements Runnable {
   private static void verifyHostIsSubordinate(HostResource host) {
     if (!host.isSubordinate()) {
       throw new BadRequestException(
-          String.format("%s isn't a subordinate hostname", host.getFullyQualifiedHostName()));
+          String.format("%s isn't a subordinate hostname", host.getHostName()));
     }
   }
 }

core/src/main/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java

@@ -154,7 +154,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
     }
 
     // Construct NS records (if any).
-    Set<String> nameserverData = domainBase.get().loadNameserverFullyQualifiedHostNames();
+    Set<String> nameserverData = domainBase.get().loadNameserverHostNames();
     Set<String> subordinateHosts = domainBase.get().getSubordinateHosts();
     if (!nameserverData.isEmpty()) {
       HashSet<String> nsRrData = new HashSet<>();

core/src/main/java/google/registry/dns/writer/dnsupdate/DnsMessageTransport.java

@@ -84,7 +84,7 @@ public class DnsMessageTransport {
    * @param query a message to send
    * @return the response received from the server
    * @throws IOException if the Socket input/output streams throws one
-   * @throws IllegalArgumentException if the query is too large to be sent (> 65535 bytes)
+   * @throws IllegalArgumentException if the query is too large to be sent (> 65535 bytes)
    */
   public Message send(Message query) throws IOException {
     try (Socket socket = factory.createSocket(InetAddress.getByName(updateHost), DNS_PORT)) {

core/src/main/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriter.java

@@ -189,7 +189,7 @@ public class DnsUpdateWriter extends BaseDnsWriter {
     for (DelegationSignerData signerData : domain.getDsData()) {
       DSRecord dsRecord =
           new DSRecord(
-              toAbsoluteName(domain.getFullyQualifiedDomainName()),
+              toAbsoluteName(domain.getDomainName()),
               DClass.IN,
               dnsDefaultDsTtl.getStandardSeconds(),
               signerData.getKeyTag(),
@@ -216,7 +216,7 @@ public class DnsUpdateWriter extends BaseDnsWriter {
   private void addInBailiwickNameServerSet(DomainBase domain, Update update) {
     for (String hostName :
         intersection(
-            domain.loadNameserverFullyQualifiedHostNames(), domain.getSubordinateHosts())) {
+            domain.loadNameserverHostNames(), domain.getSubordinateHosts())) {
       Optional<HostResource> host = loadByForeignKey(HostResource.class, hostName, clock.nowUtc());
       checkState(host.isPresent(), "Host %s cannot be loaded", hostName);
       update.add(makeAddressSet(host.get()));
@@ -226,10 +226,10 @@ public class DnsUpdateWriter extends BaseDnsWriter {
 
   private RRset makeNameServerSet(DomainBase domain) {
     RRset nameServerSet = new RRset();
-    for (String hostName : domain.loadNameserverFullyQualifiedHostNames()) {
+    for (String hostName : domain.loadNameserverHostNames()) {
       NSRecord record =
           new NSRecord(
-              toAbsoluteName(domain.getFullyQualifiedDomainName()),
+              toAbsoluteName(domain.getDomainName()),
               DClass.IN,
               dnsDefaultNsTtl.getStandardSeconds(),
               toAbsoluteName(hostName));
@@ -244,7 +244,7 @@ public class DnsUpdateWriter extends BaseDnsWriter {
       if (address instanceof Inet4Address) {
         ARecord record =
             new ARecord(
-                toAbsoluteName(host.getFullyQualifiedHostName()),
+                toAbsoluteName(host.getHostName()),
                 DClass.IN,
                 dnsDefaultATtl.getStandardSeconds(),
                 address);
@@ -260,7 +260,7 @@ public class DnsUpdateWriter extends BaseDnsWriter {
       if (address instanceof Inet6Address) {
         AAAARecord record =
             new AAAARecord(
-                toAbsoluteName(host.getFullyQualifiedHostName()),
+                toAbsoluteName(host.getHostName()),
                 DClass.IN,
                 dnsDefaultATtl.getStandardSeconds(),
                 address);

core/src/main/java/google/registry/export/BigqueryPollJobAction.java

@@ -73,8 +73,10 @@ public class BigqueryPollJobAction implements Runnable {
 
   @Override
   public void run() {
-    checkJobOutcome();  // Throws a NotModifiedException if the job hasn't completed.
-    if (payload == null || payload.length == 0) {
+    boolean jobOutcome =
+        checkJobOutcome(); // Throws a NotModifiedException if the job hasn't completed.
+    // If the job failed, do not enqueue the next step.
+    if (!jobOutcome || payload == null || payload.length == 0) {
       return;
     }
     // If there is a payload, it's a chained task, so enqueue it.

core/src/main/java/google/registry/export/ExportDomainListsAction.java

@@ -107,7 +107,7 @@ public class ExportDomainListsAction implements Runnable {
     @Override
     public void map(DomainBase domain) {
       if (realTlds.contains(domain.getTld()) && isActive(domain, exportTime)) {
-        emit(domain.getTld(), domain.getFullyQualifiedDomainName());
+        emit(domain.getTld(), domain.getDomainName());
         getContext().incrementCounter(String.format("domains in tld %s", domain.getTld()));
       }
     }

core/src/main/java/google/registry/flows/EppXmlSanitizer.java

@@ -42,8 +42,8 @@ import javax.xml.stream.events.XMLEvent;
 /**
  * Sanitizes sensitive data in incoming/outgoing EPP XML messages.
  *
- * <p>Current implementation masks user credentials (text following <pw> and <newPW> tags) as
- * follows:
+ * <p>Current implementation masks user credentials (text following &lt;pw&gt; and &lt;newPW&gt;
+ * tags) as follows:
  *
  * <ul>
  *   <li>A control character (in ranges [0 - 1F] and [7F - 9F]) is replaced with 'C'.

core/src/main/java/google/registry/flows/ResourceFlowUtils.java

@@ -48,6 +48,7 @@ import google.registry.model.eppcommon.AuthInfo;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -93,11 +94,12 @@ public final class ResourceFlowUtils {
                    * trust the query and need to do the full mapreduce.
                    */
                   Iterable<Key<DomainBase>> keys =
-                      queryForLinkedDomains(fki.getResourceKey(), now)
+                      queryForLinkedDomains(fki.getResourceKey().getOfyKey(), now)
                           .limit(FAILFAST_CHECK_COUNT)
                           .keys();
+                  VKey<R> resourceVKey = fki.getResourceKey();
                   Predicate<DomainBase> predicate =
-                      domain -> getPotentialReferences.apply(domain).contains(fki.getResourceKey());
+                      domain -> getPotentialReferences.apply(domain).contains(resourceVKey);
                   return ofy().load().keys(keys).values().stream().anyMatch(predicate)
                       ? new ResourceToDeleteIsReferencedException()
                       : null;
@@ -134,9 +136,9 @@ public final class ResourceFlowUtils {
 
   public static <R extends EppResource> void verifyResourceDoesNotExist(
       Class<R> clazz, String targetId, DateTime now, String clientId) throws EppException {
-    Key<R> key = loadAndGetKey(clazz, targetId, now);
+    VKey<R> key = loadAndGetKey(clazz, targetId, now);
     if (key != null) {
-      R resource = ofy().load().key(key).now();
+      R resource = tm().load(key);
       // These are similar exceptions, but we can track them internally as log-based metrics.
       if (Objects.equals(clientId, resource.getPersistedCurrentSponsorClientId())) {
         throw new ResourceAlreadyExistsForThisClientException(targetId);
@@ -184,9 +186,8 @@ public final class ResourceFlowUtils {
     }
     // The roid should match one of the contacts.
     Optional<Key<ContactResource>> foundContact =
-        domain
-            .getReferencedContacts()
-            .stream()
+        domain.getReferencedContacts().stream()
+            .map(VKey::getOfyKey)
             .filter(key -> key.getName().equals(authRepoId))
             .findFirst();
     if (!foundContact.isPresent()) {

core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java

@@ -97,7 +97,7 @@ public final class ContactTransferApproveFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, gainingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java

@@ -93,7 +93,7 @@ public final class ContactTransferCancelFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, losingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java

@@ -90,7 +90,7 @@ public final class ContactTransferRejectFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, gainingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java

@@ -46,7 +46,7 @@ import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.model.transfer.TransferData;
+import google.registry.model.transfer.ContactTransferData;
 import google.registry.model.transfer.TransferStatus;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -112,27 +112,30 @@ public final class ContactTransferRequestFlow implements TransactionalFlow {
         .setParent(Key.create(existingContact))
         .build();
     DateTime transferExpirationTime = now.plus(automaticTransferLength);
-    TransferData serverApproveTransferData = new TransferData.Builder()
-        .setTransferRequestTime(now)
-        .setTransferRequestTrid(trid)
-        .setGainingClientId(gainingClientId)
-        .setLosingClientId(losingClientId)
-        .setPendingTransferExpirationTime(transferExpirationTime)
-        .setTransferStatus(TransferStatus.SERVER_APPROVED)
-        .build();
+    ContactTransferData serverApproveTransferData =
+        new ContactTransferData.Builder()
+            .setTransferRequestTime(now)
+            .setTransferRequestTrid(trid)
+            .setGainingClientId(gainingClientId)
+            .setLosingClientId(losingClientId)
+            .setPendingTransferExpirationTime(transferExpirationTime)
+            .setTransferStatus(TransferStatus.SERVER_APPROVED)
+            .build();
     // If the transfer is server approved, this message will be sent to the losing registrar. */
     PollMessage serverApproveLosingPollMessage =
         createLosingTransferPollMessage(targetId, serverApproveTransferData, historyEntry);
     // If the transfer is server approved, this message will be sent to the gaining registrar. */
     PollMessage serverApproveGainingPollMessage =
         createGainingTransferPollMessage(targetId, serverApproveTransferData, historyEntry);
-    TransferData pendingTransferData = serverApproveTransferData.asBuilder()
-        .setTransferStatus(TransferStatus.PENDING)
-        .setServerApproveEntities(
-            ImmutableSet.of(
-                Key.create(serverApproveGainingPollMessage),
-                Key.create(serverApproveLosingPollMessage)))
-        .build();
+    ContactTransferData pendingTransferData =
+        serverApproveTransferData
+            .asBuilder()
+            .setTransferStatus(TransferStatus.PENDING)
+            .setServerApproveEntities(
+                ImmutableSet.of(
+                    serverApproveGainingPollMessage.createVKey(),
+                    serverApproveLosingPollMessage.createVKey()))
+            .build();
     // When a transfer is requested, a poll message is created to notify the losing registrar.
     PollMessage requestPollMessage =
         createLosingTransferPollMessage(targetId, pendingTransferData, historyEntry).asBuilder()

core/src/main/java/google/registry/flows/custom/DomainCheckFlowCustomLogic.java

@@ -75,7 +75,7 @@ public class DomainCheckFlowCustomLogic extends BaseFlowCustomLogic {
 
     /**
      * The time to perform the domain check as of. This defaults to the current time, but can be
-     * overridden in v>=0.12 of the fee extension.
+     * overridden in v>=0.12 of the fee extension.
      */
     public abstract DateTime asOfDate();
 
@@ -105,7 +105,7 @@ public class DomainCheckFlowCustomLogic extends BaseFlowCustomLogic {
 
     /**
      * The time to perform the domain check as of. This defaults to the current time, but can be
-     * overridden in v>=0.12 of the fee extension.
+     * overridden in v>=0.12 of the fee extension.
      */
     public abstract DateTime asOfDate();
 

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -354,12 +354,10 @@ public class DomainCreateFlow implements TransactionalFlow {
             .setDsData(secDnsCreate.isPresent() ? secDnsCreate.get().getDsData() : null)
             .setRegistrant(command.getRegistrant())
             .setAuthInfo(command.getAuthInfo())
-            .setFullyQualifiedDomainName(targetId)
+            .setDomainName(targetId)
             .setNameservers(
                 (ImmutableSet<VKey<HostResource>>)
-                    command.getNameservers().stream()
-                        .map(key -> VKey.createOfy(HostResource.class, key))
-                        .collect(toImmutableSet()))
+                    command.getNameservers().stream().collect(toImmutableSet()))
             .setStatusValues(statuses.build())
             .setContacts(command.getContacts())
             .addGracePeriod(GracePeriod.forBillingEvent(GracePeriodStatus.ADD, createBillingEvent))
@@ -533,7 +531,7 @@ public class DomainCreateFlow implements TransactionalFlow {
         .setPeriodYears(years)
         .setCost(feesAndCredits.getCreateCost())
         .setEventTime(now)
-        .setAllocationToken(allocationToken.map(Key::create).orElse(null))
+        .setAllocationToken(allocationToken.map(AllocationToken::createVKey).orElse(null))
         .setBillingTime(
             now.plus(
                 isAnchorTenant
@@ -600,7 +598,7 @@ public class DomainCreateFlow implements TransactionalFlow {
   private void enqueueTasks(
       DomainBase newDomain, boolean hasSignedMarks, boolean hasClaimsNotice) {
     if (newDomain.shouldPublishToDns()) {
-      dnsQueue.addDomainRefreshTask(newDomain.getFullyQualifiedDomainName());
+      dnsQueue.addDomainRefreshTask(newDomain.getDomainName());
     }
     if (hasClaimsNotice || hasSignedMarks) {
       LordnTaskUtils.enqueueDomainBaseTask(newDomain);

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -212,6 +212,28 @@ public final class DomainDeleteFlow implements TransactionalFlow {
       builder.setDeletePollMessage(Key.create(deletePollMessage));
     }
 
+    // Cancel any grace periods that were still active, and set the expiration time accordingly.
+    DateTime newExpirationTime = existingDomain.getRegistrationExpirationTime();
+    for (GracePeriod gracePeriod : existingDomain.getGracePeriods()) {
+      // No cancellation is written if the grace period was not for a billable event.
+      if (gracePeriod.hasBillingEvent()) {
+        entitiesToSave.add(
+            BillingEvent.Cancellation.forGracePeriod(gracePeriod, historyEntry, targetId));
+        if (gracePeriod.getOneTimeBillingEvent() != null) {
+          // Take the amount of amount of registration time being refunded off the expiration time.
+          // This can be either add grace periods or renew grace periods.
+          BillingEvent.OneTime oneTime =
+              ofy().load().key(gracePeriod.getOneTimeBillingEvent()).now();
+          newExpirationTime = newExpirationTime.minusYears(oneTime.getPeriodYears());
+        } else if (gracePeriod.getRecurringBillingEvent() != null) {
+          // Take 1 year off the registration if in the autorenew grace period (no need to load the
+          // recurring billing event; all autorenews are for 1 year).
+          newExpirationTime = newExpirationTime.minusYears(1);
+        }
+      }
+    }
+    builder.setRegistrationExpirationTime(newExpirationTime);
+
     DomainBase newDomain = builder.build();
     updateForeignKeyIndexDeletionTime(newDomain);
     handlePendingTransferOnDelete(existingDomain, newDomain, now, historyEntry);
@@ -220,15 +242,8 @@ public final class DomainDeleteFlow implements TransactionalFlow {
     // If there's a pending transfer, the gaining client's autorenew billing
     // event and poll message will already have been deleted in
     // ResourceDeleteFlow since it's listed in serverApproveEntities.
-    dnsQueue.addDomainRefreshTask(existingDomain.getFullyQualifiedDomainName());
-    // Cancel any grace periods that were still active.
-    for (GracePeriod gracePeriod : existingDomain.getGracePeriods()) {
-      // No cancellation is written if the grace period was not for a billable event.
-      if (gracePeriod.hasBillingEvent()) {
-        entitiesToSave.add(
-            BillingEvent.Cancellation.forGracePeriod(gracePeriod, historyEntry, targetId));
-      }
-    }
+    dnsQueue.addDomainRefreshTask(existingDomain.getDomainName());
+
     entitiesToSave.add(newDomain, historyEntry);
     EntityChanges entityChanges = flowCustomLogic.beforeSave(
         BeforeSaveParameters.newBuilder()
@@ -324,7 +339,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
         .setResponseData(
             ImmutableList.of(
                 DomainPendingActionNotificationResponse.create(
-                    existingDomain.getFullyQualifiedDomainName(), true, trid, deletionTime)))
+                    existingDomain.getDomainName(), true, trid, deletionTime)))
         .setParent(historyEntry)
         .build();
   }

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -35,6 +35,7 @@ import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRIS
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_ANCHOR_TENANT;
 import static google.registry.model.registry.label.ReservationType.RESERVED_FOR_SPECIFIC_USE;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.pricing.PricingEngineProxy.isDomainPremium;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
@@ -120,6 +121,7 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tmch.ClaimsListShard;
+import google.registry.persistence.VKey;
 import google.registry.tldconfig.idn.IdnLabelValidator;
 import google.registry.util.Idn;
 import java.math.BigDecimal;
@@ -305,10 +307,10 @@ public class DomainFlowUtils {
   /** Verify that no linked resources have disallowed statuses. */
   static void verifyNotInPendingDelete(
       Set<DesignatedContact> contacts,
-      Key<ContactResource> registrant,
-      Set<Key<HostResource>> nameservers)
+      VKey<ContactResource> registrant,
+      Set<VKey<HostResource>> nameservers)
       throws EppException {
-    ImmutableList.Builder<Key<? extends EppResource>> keysToLoad = new ImmutableList.Builder<>();
+    ImmutableList.Builder<VKey<? extends EppResource>> keysToLoad = new ImmutableList.Builder<>();
     contacts.stream().map(DesignatedContact::getContactKey).forEach(keysToLoad::add);
     Optional.ofNullable(registrant).ifPresent(keysToLoad::add);
     keysToLoad.addAll(nameservers);
@@ -355,7 +357,7 @@ public class DomainFlowUtils {
         contacts.stream()
             .collect(
                 toImmutableSetMultimap(
-                    DesignatedContact::getType, DesignatedContact::getContactKey));
+                    DesignatedContact::getType, contact -> contact.getContactKey().getOfyKey()));
 
     // If any contact type has multiple contacts:
     if (contactsByType.asMap().values().stream().anyMatch(v -> v.size() > 1)) {
@@ -376,7 +378,7 @@ public class DomainFlowUtils {
   }
 
   static void validateRequiredContactsPresent(
-      @Nullable Key<ContactResource> registrant, Set<DesignatedContact> contacts)
+      @Nullable VKey<ContactResource> registrant, Set<DesignatedContact> contacts)
       throws RequiredParameterMissingException {
     if (registrant == null) {
       throw new MissingRegistrantException();
@@ -487,7 +489,7 @@ public class DomainFlowUtils {
     return new BillingEvent.Recurring.Builder()
         .setReason(Reason.RENEW)
         .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
-        .setTargetId(domain.getFullyQualifiedDomainName())
+        .setTargetId(domain.getDomainName())
         .setClientId(domain.getCurrentSponsorClientId())
         .setEventTime(domain.getRegistrationExpirationTime());
   }
@@ -498,7 +500,7 @@ public class DomainFlowUtils {
    */
   public static PollMessage.Autorenew.Builder newAutorenewPollMessage(DomainBase domain) {
     return new PollMessage.Autorenew.Builder()
-        .setTargetId(domain.getFullyQualifiedDomainName())
+        .setTargetId(domain.getDomainName())
         .setClientId(domain.getCurrentSponsorClientId())
         .setEventTime(domain.getRegistrationExpirationTime())
         .setMsg("Domain was auto-renewed.");
@@ -688,7 +690,7 @@ public class DomainFlowUtils {
     List<Fee> fees = feeCommand.get().getFees();
     // The schema guarantees that at least one fee will be present.
     checkState(!fees.isEmpty());
-    BigDecimal total = BigDecimal.ZERO;
+    BigDecimal total = zeroInCurrency(feeCommand.get().getCurrency());
     for (Fee fee : fees) {
       if (!fee.hasDefaultAttributes()) {
         throw new UnsupportedFeeAttributeException();
@@ -936,6 +938,16 @@ public class DomainFlowUtils {
     }
   }
 
+  /**
+   * Returns zero for a specific currency.
+   *
+   * <p>{@link BigDecimal} has a concept of significant figures, so zero is not always zero. E.g.
+   * zero in USD is 0.00, whereas zero in Yen is 0, and zero in Dinars is 0.000 (!).
+   */
+  static BigDecimal zeroInCurrency(CurrencyUnit currencyUnit) {
+    return Money.of(currencyUnit, BigDecimal.ZERO).getAmount();
+  }
+
   /**
    * Check that if there's a claims notice it's on the claims list, and that if there's not one it's
    * not on the claims list.
@@ -978,7 +990,7 @@ public class DomainFlowUtils {
     for (DesignatedContact contact : contacts) {
       builder.add(
           ForeignKeyedDesignatedContact.create(
-              contact.getType(), ofy().load().key(contact.getContactKey()).now().getContactId()));
+              contact.getType(), tm().load(contact.getContactKey()).getContactId()));
     }
     return builder.build();
   }

core/src/main/java/google/registry/flows/domain/DomainInfoFlow.java

@@ -21,7 +21,6 @@ import static google.registry.flows.domain.DomainFlowUtils.addSecDnsExtensionIfP
 import static google.registry.flows.domain.DomainFlowUtils.handleFeeRequest;
 import static google.registry.flows.domain.DomainFlowUtils.loadForeignKeyedDesignatedContacts;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
@@ -102,16 +101,16 @@ public final class DomainInfoFlow implements Flow {
     flowCustomLogic.afterValidation(
         AfterValidationParameters.newBuilder().setDomain(domain).build());
     // Prefetch all referenced resources. Calling values() blocks until loading is done.
-    // We do nameservers separately since they've been converted to VKey.
     tm().load(domain.getNameservers());
-    ofy().load().values(domain.getReferencedContacts()).values();
+    tm().load(domain.getReferencedContacts());
     // Registrars can only see a few fields on unauthorized domains.
     // This is a policy decision that is left up to us by the rfcs.
-    DomainInfoData.Builder infoBuilder = DomainInfoData.newBuilder()
-        .setFullyQualifiedDomainName(domain.getFullyQualifiedDomainName())
-        .setRepoId(domain.getRepoId())
-        .setCurrentSponsorClientId(domain.getCurrentSponsorClientId())
-        .setRegistrant(ofy().load().key(domain.getRegistrant()).now().getContactId());
+    DomainInfoData.Builder infoBuilder =
+        DomainInfoData.newBuilder()
+            .setFullyQualifiedDomainName(domain.getDomainName())
+            .setRepoId(domain.getRepoId())
+            .setCurrentSponsorClientId(domain.getCurrentSponsorClientId())
+            .setRegistrant(tm().load(domain.getRegistrant()).getContactId());
     // If authInfo is non-null, then the caller is authorized to see the full information since we
     // will have already verified the authInfo is valid.
     if (clientId.equals(domain.getCurrentSponsorClientId()) || authInfo.isPresent()) {
@@ -120,7 +119,7 @@ public final class DomainInfoFlow implements Flow {
           .setStatusValues(domain.getStatusValues())
           .setContacts(loadForeignKeyedDesignatedContacts(domain.getContacts()))
           .setNameservers(hostsRequest.requestDelegated()
-              ? domain.loadNameserverFullyQualifiedHostNames()
+              ? domain.loadNameserverHostNames()
               : null)
           .setSubordinateHosts(hostsRequest.requestSubordinate()
               ? domain.getSubordinateHosts()

core/src/main/java/google/registry/flows/domain/DomainPricingLogic.java

@@ -14,8 +14,9 @@
 
 package google.registry.flows.domain;
 
+import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
 import static google.registry.pricing.PricingEngineProxy.getDomainFeeClass;
-import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
+import static google.registry.pricing.PricingEngineProxy.getPricesForDomainName;
 
 import com.google.common.net.InternetDomainName;
 import google.registry.flows.EppException;
@@ -33,7 +34,6 @@ import google.registry.model.domain.fee.Fee;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.pricing.PremiumPricingEngine.DomainPrices;
 import google.registry.model.registry.Registry;
-import google.registry.pricing.PricingEngineProxy;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.util.Optional;
@@ -64,21 +64,27 @@ public final class DomainPricingLogic {
   public FeesAndCredits getCreatePrice(
       Registry registry,
       String domainName,
-      DateTime date,
+      DateTime dateTime,
       int years,
       boolean isAnchorTenant,
       Optional<AllocationToken> allocationToken)
       throws EppException {
     CurrencyUnit currency = registry.getCurrency();
+
+    BaseFee createFeeOrCredit;
     // Domain create cost is always zero for anchor tenants
-    Money domainCreateCost =
-        isAnchorTenant
-            ? Money.of(currency, BigDecimal.ZERO)
-            : getDomainCreateCostWithDiscount(domainName, date, years, allocationToken);
-    BaseFee createFeeOrCredit = Fee.create(domainCreateCost.getAmount(), FeeType.CREATE);
+    if (isAnchorTenant) {
+      createFeeOrCredit = Fee.create(zeroInCurrency(currency), FeeType.CREATE, false);
+    } else {
+      DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
+      Money domainCreateCost =
+          getDomainCreateCostWithDiscount(domainPrices, years, allocationToken);
+      createFeeOrCredit =
+          Fee.create(domainCreateCost.getAmount(), FeeType.CREATE, domainPrices.isPremium());
+    }
 
     // Create fees for the cost and the EAP fee, if any.
-    Fee eapFee = registry.getEapFeeFor(date);
+    Fee eapFee = registry.getEapFeeFor(dateTime);
     FeesAndCredits.Builder feesBuilder =
         new FeesAndCredits.Builder().setCurrency(currency).addFeeOrCredit(createFeeOrCredit);
     // Don't charge anchor tenants EAP fees.
@@ -92,7 +98,7 @@ public final class DomainPricingLogic {
             .setFeesAndCredits(feesBuilder.build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .setYears(years)
             .build());
   }
@@ -100,69 +106,73 @@ public final class DomainPricingLogic {
   /** Returns a new renew price for the pricer. */
   @SuppressWarnings("unused")
   public FeesAndCredits getRenewPrice(
-      Registry registry,
-      String domainName,
-      DateTime date,
-      int years)
-      throws EppException {
-    Money renewCost = getDomainRenewCost(domainName, date, years);
+      Registry registry, String domainName, DateTime dateTime, int years) throws EppException {
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
+    BigDecimal renewCost = domainPrices.getRenewCost().multipliedBy(years).getAmount();
     return customLogic.customizeRenewPrice(
         RenewPriceParameters.newBuilder()
             .setFeesAndCredits(
                 new FeesAndCredits.Builder()
                     .setCurrency(registry.getCurrency())
-                    .addFeeOrCredit(Fee.create(renewCost.getAmount(), FeeType.RENEW))
+                    .addFeeOrCredit(Fee.create(renewCost, FeeType.RENEW, domainPrices.isPremium()))
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .setYears(years)
             .build());
   }
 
   /** Returns a new restore price for the pricer. */
-  public FeesAndCredits getRestorePrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getRestorePrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
     FeesAndCredits feesAndCredits =
         new FeesAndCredits.Builder()
             .setCurrency(registry.getCurrency())
             .addFeeOrCredit(
-                Fee.create(getDomainRenewCost(domainName, date, 1).getAmount(), FeeType.RENEW))
+                Fee.create(
+                    domainPrices.getRenewCost().getAmount(),
+                    FeeType.RENEW,
+                    domainPrices.isPremium()))
             .addFeeOrCredit(
-                Fee.create(registry.getStandardRestoreCost().getAmount(), FeeType.RESTORE))
+                Fee.create(registry.getStandardRestoreCost().getAmount(), FeeType.RESTORE, false))
             .build();
     return customLogic.customizeRestorePrice(
         RestorePriceParameters.newBuilder()
             .setFeesAndCredits(feesAndCredits)
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns a new transfer price for the pricer. */
-  public FeesAndCredits getTransferPrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getTransferPrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
-    Money renewCost = getDomainRenewCost(domainName, date, 1);
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
     return customLogic.customizeTransferPrice(
         TransferPriceParameters.newBuilder()
             .setFeesAndCredits(
                 new FeesAndCredits.Builder()
                     .setCurrency(registry.getCurrency())
-                    .addFeeOrCredit(Fee.create(renewCost.getAmount(), FeeType.RENEW))
+                    .addFeeOrCredit(
+                        Fee.create(
+                            domainPrices.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            domainPrices.isPremium()))
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns a new update price for the pricer. */
-  public FeesAndCredits getUpdatePrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getUpdatePrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
     CurrencyUnit currency = registry.getCurrency();
-    BaseFee feeOrCredit =
-        Fee.create(Money.zero(registry.getCurrency()).getAmount(), FeeType.UPDATE);
+    BaseFee feeOrCredit = Fee.create(zeroInCurrency(currency), FeeType.UPDATE, false);
     return customLogic.customizeUpdatePrice(
         UpdatePriceParameters.newBuilder()
             .setFeesAndCredits(
@@ -172,19 +182,19 @@ public final class DomainPricingLogic {
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns the fee class for a given domain and date. */
-  public Optional<String> getFeeClass(String domainName, DateTime date) {
-    return getDomainFeeClass(domainName, date);
+  public Optional<String> getFeeClass(String domainName, DateTime dateTime) {
+    return getDomainFeeClass(domainName, dateTime);
   }
 
+  /** Returns the domain create cost with allocation-token-related discounts applied. */
   private Money getDomainCreateCostWithDiscount(
-      String domainName, DateTime date, int years, Optional<AllocationToken> allocationToken)
+      DomainPrices domainPrices, int years, Optional<AllocationToken> allocationToken)
       throws EppException {
-    DomainPrices domainPrices = PricingEngineProxy.getPricesForDomainName(domainName, date);
     if (allocationToken.isPresent()
         && allocationToken.get().getDiscountFraction() != 0.0
         && domainPrices.isPremium()) {

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -211,8 +211,7 @@ public final class DomainRenewFlow implements TransactionalFlow {
             BeforeResponseParameters.newBuilder()
                 .setDomain(newDomain)
                 .setResData(DomainRenewData.create(targetId, newExpirationTime))
-                .setResponseExtensions(
-                    createResponseExtensions(feesAndCredits.getTotalCost(), feeRenew))
+                .setResponseExtensions(createResponseExtensions(feesAndCredits, feeRenew))
                 .build());
     return responseBuilder
         .setResData(responseData.resData())
@@ -270,14 +269,19 @@ public final class DomainRenewFlow implements TransactionalFlow {
   }
 
   private ImmutableList<FeeTransformResponseExtension> createResponseExtensions(
-      Money renewCost, Optional<FeeRenewCommandExtension> feeRenew) {
+      FeesAndCredits feesAndCredits, Optional<FeeRenewCommandExtension> feeRenew) {
     return feeRenew.isPresent()
         ? ImmutableList.of(
             feeRenew
                 .get()
                 .createResponseBuilder()
-                .setCurrency(renewCost.getCurrencyUnit())
-                .setFees(ImmutableList.of(Fee.create(renewCost.getAmount(), FeeType.RENEW)))
+                .setCurrency(feesAndCredits.getCurrency())
+                .setFees(
+                    ImmutableList.of(
+                        Fee.create(
+                            feesAndCredits.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RENEW))))
                 .build())
         : ImmutableList.of();
   }

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -143,24 +143,30 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     verifyRestoreAllowed(command, existingDomain, feeUpdate, feesAndCredits, now);
     HistoryEntry historyEntry = buildHistoryEntry(existingDomain, now);
     ImmutableSet.Builder<ImmutableObject> entitiesToSave = new ImmutableSet.Builder<>();
-    entitiesToSave.addAll(
-        createRestoreAndRenewBillingEvents(
-            historyEntry, feesAndCredits.getRestoreCost(), feesAndCredits.getRenewCost(), now));
-    // We don't preserve the original expiration time of the domain when we restore, since doing so
-    // would require us to know if they received a grace period refund when they deleted the domain,
-    // and to charge them for that again. Instead, we just say that all restores get a fresh year of
-    // registration and bill them for that accordingly.
-    DateTime newExpirationTime = now.plusYears(1);
-    BillingEvent.Recurring autorenewEvent = newAutorenewBillingEvent(existingDomain)
-        .setEventTime(newExpirationTime)
-        .setRecurrenceEndTime(END_OF_TIME)
-        .setParent(historyEntry)
-        .build();
-    PollMessage.Autorenew autorenewPollMessage = newAutorenewPollMessage(existingDomain)
-        .setEventTime(newExpirationTime)
-        .setAutorenewEndTime(END_OF_TIME)
-        .setParent(historyEntry)
-        .build();
+
+    // Restore the expiration time on the deleted domain, except if that's already passed, then add
+    // a year and bill for it immediately, with no grace period.
+    DateTime newExpirationTime = existingDomain.getRegistrationExpirationTime();
+    if (newExpirationTime.isBefore(now)) {
+      entitiesToSave.add(createRenewBillingEvent(historyEntry, feesAndCredits.getRenewCost(), now));
+      newExpirationTime = newExpirationTime.plusYears(1);
+    }
+    // Always bill for the restore itself.
+    entitiesToSave.add(
+        createRestoreBillingEvent(historyEntry, feesAndCredits.getRestoreCost(), now));
+
+    BillingEvent.Recurring autorenewEvent =
+        newAutorenewBillingEvent(existingDomain)
+            .setEventTime(newExpirationTime)
+            .setRecurrenceEndTime(END_OF_TIME)
+            .setParent(historyEntry)
+            .build();
+    PollMessage.Autorenew autorenewPollMessage =
+        newAutorenewPollMessage(existingDomain)
+            .setEventTime(newExpirationTime)
+            .setAutorenewEndTime(END_OF_TIME)
+            .setParent(historyEntry)
+            .build();
     DomainBase newDomain =
         performRestore(
             existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId);
@@ -168,11 +174,9 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     entitiesToSave.add(newDomain, historyEntry, autorenewEvent, autorenewPollMessage);
     ofy().save().entities(entitiesToSave.build());
     ofy().delete().key(existingDomain.getDeletePollMessage());
-    dnsQueue.addDomainRefreshTask(existingDomain.getFullyQualifiedDomainName());
+    dnsQueue.addDomainRefreshTask(existingDomain.getDomainName());
     return responseBuilder
-        .setExtensions(
-            createResponseExtensions(
-                feesAndCredits.getRestoreCost(), feesAndCredits.getRenewCost(), feeUpdate))
+        .setExtensions(createResponseExtensions(feesAndCredits, feeUpdate))
         .build();
   }
 
@@ -212,18 +216,6 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     validateFeeChallenge(targetId, now, feeUpdate, feesAndCredits);
   }
 
-  private ImmutableSet<BillingEvent.OneTime> createRestoreAndRenewBillingEvents(
-      HistoryEntry historyEntry, Money restoreCost, Money renewCost, DateTime now) {
-    // Bill for the restore.
-    BillingEvent.OneTime restoreEvent = createRestoreBillingEvent(historyEntry, restoreCost, now);
-    // Create a new autorenew billing event and poll message starting at the new expiration time.
-    // Also bill for the 1 year cost of a domain renew. This is to avoid registrants being able to
-    // game the system for premium names by renewing, deleting, and then restoring to get a free
-    // year. Note that this billing event has no grace period; it is effective immediately.
-    BillingEvent.OneTime renewEvent = createRenewBillingEvent(historyEntry, renewCost, now);
-    return ImmutableSet.of(restoreEvent, renewEvent);
-  }
-
   private static DomainBase performRestore(
       DomainBase existingDomain,
       DateTime newExpirationTime,
@@ -271,17 +263,23 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
   }
 
   private static ImmutableList<FeeTransformResponseExtension> createResponseExtensions(
-      Money restoreCost, Money renewCost, Optional<FeeUpdateCommandExtension> feeUpdate) {
+      FeesAndCredits feesAndCredits, Optional<FeeUpdateCommandExtension> feeUpdate) {
     return feeUpdate.isPresent()
         ? ImmutableList.of(
             feeUpdate
                 .get()
                 .createResponseBuilder()
-                .setCurrency(restoreCost.getCurrencyUnit())
+                .setCurrency(feesAndCredits.getCurrency())
                 .setFees(
                     ImmutableList.of(
-                        Fee.create(restoreCost.getAmount(), FeeType.RESTORE),
-                        Fee.create(renewCost.getAmount(), FeeType.RENEW)))
+                        Fee.create(
+                            feesAndCredits.getRestoreCost().getAmount(),
+                            FeeType.RESTORE,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RESTORE)),
+                        Fee.create(
+                            feesAndCredits.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RENEW))))
                 .build())
         : ImmutableList.of();
   }

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -58,7 +58,7 @@ import google.registry.model.registry.Registry;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.model.transfer.TransferData;
+import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -112,7 +112,7 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
     if (!isSuperuser) {
       checkAllowedAccessToTld(clientId, tld);
     }
-    TransferData transferData = existingDomain.getTransferData();
+    DomainTransferData transferData = existingDomain.getTransferData();
     String gainingClientId = transferData.getGainingClientId();
     Registry registry = Registry.get(existingDomain.getTld());
     HistoryEntry historyEntry = buildHistoryEntry(existingDomain, registry, now, gainingClientId);
@@ -214,7 +214,7 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
     ofy().save().entities(entitiesToSave.build());
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(
             targetId, newDomain.getTransferData(), newDomain.getRegistrationExpirationTime()))

core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java

@@ -110,7 +110,7 @@ public final class DomainTransferCancelFlow implements TransactionalFlow {
     updateAutorenewRecurrenceEndTime(existingDomain, END_OF_TIME);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newDomain.getTransferData(), null))
         .build();

core/src/main/java/google/registry/flows/domain/DomainTransferQueryFlow.java

@@ -32,7 +32,7 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.AuthInfo;
 import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.model.transfer.TransferData;
+import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.util.Clock;
 import java.util.Optional;
@@ -74,7 +74,7 @@ public final class DomainTransferQueryFlow implements Flow {
     verifyOptionalAuthInfo(authInfo, domain);
     // Most of the fields on the transfer response are required, so there's no way to return valid
     // XML if the object has never been transferred (and hence the fields aren't populated).
-    TransferData transferData = domain.getTransferData();
+    DomainTransferData transferData = domain.getTransferData();
     if (transferData.getTransferStatus() == null) {
       throw new NoTransferHistoryToQueryException();
     }

core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java

@@ -112,7 +112,7 @@ public final class DomainTransferRejectFlow implements TransactionalFlow {
     updateAutorenewRecurrenceEndTime(existingDomain, END_OF_TIME);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newDomain.getTransferData(), null))
         .build();

core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java

@@ -69,7 +69,7 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.model.transfer.TransferData;
+import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.model.transfer.TransferStatus;
@@ -198,9 +198,9 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
             feesAndCredits.map(FeesAndCredits::getTotalCost),
             now);
     // Create the transfer data that represents the pending transfer.
-    TransferData pendingTransferData =
+    DomainTransferData pendingTransferData =
         createPendingTransferData(
-            new TransferData.Builder()
+            new DomainTransferData.Builder()
                 .setTransferRequestTrid(trid)
                 .setTransferRequestTime(now)
                 .setGainingClientId(gainingClientId)

core/src/main/java/google/registry/flows/domain/DomainTransferUtils.java

@@ -20,7 +20,6 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
@@ -33,10 +32,12 @@ import google.registry.model.poll.PendingActionNotificationResponse.DomainPendin
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import org.joda.money.Money;
@@ -48,41 +49,38 @@ import org.joda.time.DateTime;
 public final class DomainTransferUtils {
 
   /** Sets up {@link TransferData} for a domain with links to entities for server approval. */
-  public static TransferData createPendingTransferData(
-      TransferData.Builder transferDataBuilder,
+  public static DomainTransferData createPendingTransferData(
+      DomainTransferData.Builder transferDataBuilder,
       ImmutableSet<TransferServerApproveEntity> serverApproveEntities,
       Period transferPeriod) {
-    ImmutableSet.Builder<Key<? extends TransferServerApproveEntity>> serverApproveEntityKeys =
+    ImmutableSet.Builder<VKey<? extends TransferServerApproveEntity>> serverApproveEntityKeys =
         new ImmutableSet.Builder<>();
     for (TransferServerApproveEntity entity : serverApproveEntities) {
-      serverApproveEntityKeys.add(Key.create(entity));
+      serverApproveEntityKeys.add(entity.createVKey());
     }
     if (transferPeriod.getValue() != 0) {
       // Unless superuser sets period to 0, add a transfer billing event.
       transferDataBuilder.setServerApproveBillingEvent(
-          Key.create(
-              serverApproveEntities
-                  .stream()
-                  .filter(BillingEvent.OneTime.class::isInstance)
-                  .map(BillingEvent.OneTime.class::cast)
-                  .collect(onlyElement())));
+          serverApproveEntities.stream()
+              .filter(BillingEvent.OneTime.class::isInstance)
+              .map(BillingEvent.OneTime.class::cast)
+              .collect(onlyElement())
+              .createVKey());
     }
     return transferDataBuilder
         .setTransferStatus(TransferStatus.PENDING)
         .setServerApproveAutorenewEvent(
-            Key.create(
-                serverApproveEntities
-                    .stream()
-                    .filter(BillingEvent.Recurring.class::isInstance)
-                    .map(BillingEvent.Recurring.class::cast)
-                    .collect(onlyElement())))
+            serverApproveEntities.stream()
+                .filter(BillingEvent.Recurring.class::isInstance)
+                .map(BillingEvent.Recurring.class::cast)
+                .collect(onlyElement())
+                .createVKey())
         .setServerApproveAutorenewPollMessage(
-            Key.create(
-                serverApproveEntities
-                    .stream()
-                    .filter(PollMessage.Autorenew.class::isInstance)
-                    .map(PollMessage.Autorenew.class::cast)
-                    .collect(onlyElement())))
+            serverApproveEntities.stream()
+                .filter(PollMessage.Autorenew.class::isInstance)
+                .map(PollMessage.Autorenew.class::cast)
+                .collect(onlyElement())
+                .createVKey())
         .setServerApproveEntities(serverApproveEntityKeys.build())
         .setTransferPeriod(transferPeriod)
         .build();
@@ -111,10 +109,10 @@ public final class DomainTransferUtils {
       String gainingClientId,
       Optional<Money> transferCost,
       DateTime now) {
-    String targetId = existingDomain.getFullyQualifiedDomainName();
+    String targetId = existingDomain.getDomainName();
     // Create a TransferData for the server-approve case to use for the speculative poll messages.
-    TransferData serverApproveTransferData =
-        new TransferData.Builder()
+    DomainTransferData serverApproveTransferData =
+        new DomainTransferData.Builder()
             .setTransferRequestTrid(trid)
             .setTransferRequestTime(now)
             .setGainingClientId(gainingClientId)

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -72,11 +72,9 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
-import google.registry.model.host.HostResource;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.persistence.VKey;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
@@ -246,14 +244,8 @@ public final class DomainUpdateFlow implements TransactionalFlow {
             .setLastEppUpdateClientId(clientId)
             .addStatusValues(add.getStatusValues())
             .removeStatusValues(remove.getStatusValues())
-            .addNameservers(
-                add.getNameservers().stream()
-                    .map(key -> VKey.createOfy(HostResource.class, key))
-                    .collect(toImmutableSet()))
-            .removeNameservers(
-                remove.getNameservers().stream()
-                    .map(key -> VKey.createOfy(HostResource.class, key))
-                    .collect(toImmutableSet()))
+            .addNameservers(add.getNameservers().stream().collect(toImmutableSet()))
+            .removeNameservers(remove.getNameservers().stream().collect(toImmutableSet()))
             .addContacts(add.getContacts())
             .removeContacts(remove.getContacts())
             .setRegistrant(firstNonNull(change.getRegistrant(), domain.getRegistrant()))
@@ -273,7 +265,7 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     validateDsData(newDomain.getDsData());
     validateNameserversCountForTld(
         newDomain.getTld(),
-        InternetDomainName.from(newDomain.getFullyQualifiedDomainName()),
+        InternetDomainName.from(newDomain.getDomainName()),
         newDomain.getNameservers().size());
   }
 

core/src/main/java/google/registry/flows/domain/FeesAndCredits.java

@@ -15,6 +15,7 @@
 package google.registry.flows.domain;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
@@ -27,6 +28,7 @@ import google.registry.model.domain.fee.BaseFee;
 import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Credit;
 import google.registry.model.domain.fee.Fee;
+import java.math.BigDecimal;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 
@@ -39,26 +41,30 @@ public class FeesAndCredits extends ImmutableObject implements Buildable {
   private ImmutableList<Credit> credits;
 
   private Money getTotalCostForType(FeeType type) {
-    Money result = Money.zero(currency);
     checkArgumentNotNull(type);
-    for (Fee fee : fees) {
-      if (fee.getType() == type) {
-        result = result.plus(fee.getCost());
-      }
-    }
-    return result;
+    return Money.of(
+        currency,
+        fees.stream()
+            .filter(f -> f.getType() == type)
+            .map(BaseFee::getCost)
+            .reduce(zeroInCurrency(currency), BigDecimal::add));
+  }
+
+  public boolean hasPremiumFeesOfType(FeeType type) {
+    return fees.stream().filter(f -> f.getType() == type).anyMatch(BaseFee::isPremium);
   }
 
   /** Returns the total cost of all fees and credits for the event. */
   public Money getTotalCost() {
-    Money result = Money.zero(currency);
-    for (Fee fee : fees) {
-      result = result.plus(fee.getCost());
-    }
-    for (Credit credit : credits) {
-      result = result.plus(credit.getCost());
-    }
-    return result;
+    return Money.of(
+        currency,
+        Streams.concat(fees.stream(), credits.stream())
+            .map(BaseFee::getCost)
+            .reduce(zeroInCurrency(currency), BigDecimal::add));
+  }
+
+  public boolean hasAnyPremiumFees() {
+    return fees.stream().anyMatch(BaseFee::isPremium);
   }
 
   /** Returns the create cost for the event. */

core/src/main/java/google/registry/flows/host/HostCreateFlow.java

@@ -125,10 +125,10 @@ public final class HostCreateFlow implements TransactionalFlow {
         new HostResource.Builder()
             .setCreationClientId(clientId)
             .setPersistedCurrentSponsorClientId(clientId)
-            .setFullyQualifiedHostName(targetId)
+            .setHostName(targetId)
             .setInetAddresses(command.getInetAddresses())
             .setRepoId(createRepoId(ObjectifyService.allocateId(), roidSuffix))
-            .setSuperordinateDomain(superordinateDomain.map(Key::create).orElse(null))
+            .setSuperordinateDomain(superordinateDomain.map(DomainBase::createVKey).orElse(null))
             .build();
     historyBuilder
         .setType(HistoryEntry.Type.HOST_CREATE)

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -14,7 +14,6 @@
 
 package google.registry.flows.host;
 
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.ResourceFlowUtils.failfastForAsyncDelete;
 import static google.registry.flows.ResourceFlowUtils.loadAndVerifyExistence;
@@ -82,17 +81,6 @@ public final class HostDeleteFlow implements TransactionalFlow {
   @Inject EppResponse.Builder responseBuilder;
   @Inject HostDeleteFlow() {}
 
-  /**
-   * Hack to convert DomainBase's nameserver VKey's to Ofy Key's.
-   *
-   * <p>We currently need this because {@code failfastForAsyncDelete()} checks to see if a name is
-   * in the ofy keys and is used for both nameservers and contacts. When we convert contacts to
-   * VKey's, we can remove this and do the conversion in {@code failfastForAsyncDelete()}.
-   */
-  private static ImmutableSet<Key<HostResource>> getNameserverOfyKeys(DomainBase domain) {
-    return domain.getNameservers().stream().map(key -> key.getOfyKey()).collect(toImmutableSet());
-  }
-
   @Override
   public final EppResponse run() throws EppException {
     extensionManager.register(MetadataExtension.class);
@@ -100,7 +88,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
     validateClientIsLoggedIn(clientId);
     DateTime now = tm().getTransactionTime();
     validateHostName(targetId);
-    failfastForAsyncDelete(targetId, now, HostResource.class, HostDeleteFlow::getNameserverOfyKeys);
+    failfastForAsyncDelete(targetId, now, HostResource.class, DomainBase::getNameservers);
     HostResource existingHost = loadAndVerifyExistence(HostResource.class, targetId, now);
     verifyNoDisallowedStatuses(existingHost, DISALLOWED_STATUSES);
     if (!isSuperuser) {
@@ -108,8 +96,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
       // the client id, needs to be read off of it.
       EppResource owningResource =
           existingHost.isSubordinate()
-              ? ofy().load().key(existingHost.getSuperordinateDomain()).now()
-                  .cloneProjectedAtTime(now)
+              ? tm().load(existingHost.getSuperordinateDomain()).cloneProjectedAtTime(now)
               : existingHost;
       verifyResourceOwnership(clientId, owningResource);
     }

core/src/main/java/google/registry/flows/host/HostInfoFlow.java

@@ -18,7 +18,7 @@ import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.ResourceFlowUtils.loadAndVerifyExistence;
 import static google.registry.flows.host.HostFlowUtils.validateHostName;
 import static google.registry.model.EppResourceUtils.isLinked;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
@@ -77,7 +77,7 @@ public final class HostInfoFlow implements Flow {
     // there is no superordinate domain, the host's own values for these fields will be correct.
     if (host.isSubordinate()) {
       DomainBase superordinateDomain =
-          ofy().load().key(host.getSuperordinateDomain()).now().cloneProjectedAtTime(now);
+          tm().load(host.getSuperordinateDomain()).cloneProjectedAtTime(now);
       hostInfoDataBuilder
           .setCurrentSponsorClientId(superordinateDomain.getCurrentSponsorClientId())
           .setLastTransferTime(host.computeLastTransferTime(superordinateDomain));
@@ -91,7 +91,7 @@ public final class HostInfoFlow implements Flow {
     }
     return responseBuilder
         .setResData(hostInfoDataBuilder
-            .setFullyQualifiedHostName(host.getFullyQualifiedHostName())
+            .setFullyQualifiedHostName(host.getHostName())
             .setRepoId(host.getRepoId())
             .setStatusValues(statusValues.build())
             .setInetAddresses(host.getInetAddresses())

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -60,6 +60,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
+import google.registry.persistence.VKey;
 import java.util.Objects;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -136,9 +137,10 @@ public final class HostUpdateFlow implements TransactionalFlow {
     boolean isHostRename = suppliedNewHostName != null;
     String oldHostName = targetId;
     String newHostName = firstNonNull(suppliedNewHostName, oldHostName);
-    DomainBase oldSuperordinateDomain = existingHost.isSubordinate()
-        ? ofy().load().key(existingHost.getSuperordinateDomain()).now().cloneProjectedAtTime(now)
-        : null;
+    DomainBase oldSuperordinateDomain =
+        existingHost.isSubordinate()
+            ? tm().load(existingHost.getSuperordinateDomain()).cloneProjectedAtTime(now)
+            : null;
     // Note that lookupSuperordinateDomain calls cloneProjectedAtTime on the domain for us.
     Optional<DomainBase> newSuperordinateDomain =
         lookupSuperordinateDomain(validateHostName(newHostName), now);
@@ -153,8 +155,8 @@ public final class HostUpdateFlow implements TransactionalFlow {
     AddRemove remove = command.getInnerRemove();
     checkSameValuesNotAddedAndRemoved(add.getStatusValues(), remove.getStatusValues());
     checkSameValuesNotAddedAndRemoved(add.getInetAddresses(), remove.getInetAddresses());
-    Key<DomainBase> newSuperordinateDomainKey =
-        newSuperordinateDomain.map(Key::create).orElse(null);
+    VKey<DomainBase> newSuperordinateDomainKey =
+        newSuperordinateDomain.map(DomainBase::createVKey).orElse(null);
     // If the superordinateDomain field is changing, set the lastSuperordinateChange to now.
     DateTime lastSuperordinateChange =
         Objects.equals(newSuperordinateDomainKey, existingHost.getSuperordinateDomain())
@@ -174,7 +176,7 @@ public final class HostUpdateFlow implements TransactionalFlow {
             ? newSuperordinateDomain.get().getCurrentSponsorClientId()
             : owningResource.getPersistedCurrentSponsorClientId();
     HostResource newHost = existingHost.asBuilder()
-        .setFullyQualifiedHostName(newHostName)
+        .setHostName(newHostName)
         .addStatusValues(add.getStatusValues())
         .removeStatusValues(remove.getStatusValues())
         .addInetAddresses(add.getInetAddresses())
@@ -261,14 +263,14 @@ public final class HostUpdateFlow implements TransactionalFlow {
     // Only update DNS for subordinate hosts. External hosts have no glue to write, so they
     // are only written as NS records from the referencing domain.
     if (existingHost.isSubordinate()) {
-      dnsQueue.addHostRefreshTask(existingHost.getFullyQualifiedHostName());
+      dnsQueue.addHostRefreshTask(existingHost.getHostName());
     }
     // In case of a rename, there are many updates we need to queue up.
     if (((Update) resourceCommand).getInnerChange().getFullyQualifiedHostName() != null) {
       // If the renamed host is also subordinate, then we must enqueue an update to write the new
       // glue.
       if (newHost.isSubordinate()) {
-        dnsQueue.addHostRefreshTask(newHost.getFullyQualifiedHostName());
+        dnsQueue.addHostRefreshTask(newHost.getHostName());
       }
       // We must also enqueue updates for all domains that use this host as their nameserver so
       // that their NS records can be updated to point at the new name.
@@ -280,28 +282,28 @@ public final class HostUpdateFlow implements TransactionalFlow {
     if (existingHost.isSubordinate()
         && newHost.isSubordinate()
         && Objects.equals(
-            existingHost.getSuperordinateDomain(),
-            newHost.getSuperordinateDomain())) {
-      ofy().save().entity(
-          ofy().load().key(existingHost.getSuperordinateDomain()).now().asBuilder()
-              .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
-              .addSubordinateHost(newHost.getFullyQualifiedHostName())
-              .build());
+            existingHost.getSuperordinateDomain(), newHost.getSuperordinateDomain())) {
+      tm().saveNewOrUpdate(
+              tm().load(existingHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .removeSubordinateHost(existingHost.getHostName())
+                  .addSubordinateHost(newHost.getHostName())
+                  .build());
       return;
     }
     if (existingHost.isSubordinate()) {
-      ofy().save().entity(
-          ofy().load().key(existingHost.getSuperordinateDomain()).now()
-              .asBuilder()
-              .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
-              .build());
+      tm().saveNewOrUpdate(
+              tm().load(existingHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .removeSubordinateHost(existingHost.getHostName())
+                  .build());
     }
     if (newHost.isSubordinate()) {
-      ofy().save().entity(
-          ofy().load().key(newHost.getSuperordinateDomain()).now()
-              .asBuilder()
-              .addSubordinateHost(newHost.getFullyQualifiedHostName())
-              .build());
+      tm().saveNewOrUpdate(
+              tm().load(newHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .addSubordinateHost(newHost.getHostName())
+                  .build());
     }
   }
 

core/src/main/java/google/registry/model/EppResource.java

@@ -16,12 +16,11 @@ package google.registry.model;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
@@ -32,11 +31,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Index;
@@ -44,12 +41,15 @@ import google.registry.config.RegistryConfig;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.transfer.TransferData;
+import google.registry.persistence.VKey;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.StreamSupport;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
@@ -58,23 +58,29 @@ import org.joda.time.Duration;
 
 /** An EPP entity object (i.e. a domain, contact, or host). */
 @MappedSuperclass
+@Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property)
 public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /**
    * Unique identifier in the registry for this resource.
    *
    * <p>This is in the (\w|_){1,80}-\w{1,8} format specified by RFC 5730 for roidType.
+   *
    * @see <a href="https://tools.ietf.org/html/rfc5730">RFC 5730</a>
    */
-  @Id @javax.persistence.Id String repoId;
+  @Id
+  // not persisted so that we can store these in references to other objects. Subclasses that wish
+  // to use this as the primary key should create a getter method annotated with @Id
+  @Transient
+  String repoId;
 
   /** The ID of the registrar that is currently sponsoring this resource. */
   @Index
-  @Column(nullable = false)
+  @Column(name = "currentSponsorRegistrarId", nullable = false)
   String currentSponsorClientId;
 
   /** The ID of the registrar that created this resource. */
-  @Column(nullable = false)
+  @Column(name = "creationRegistrarId", nullable = false)
   String creationClientId;
 
   /**
@@ -84,6 +90,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * edits; it only includes EPP-visible modifications such as {@literal <update>}. Can be null if
    * the resource has never been modified.
    */
+  @Column(name = "lastEppUpdateRegistrarId")
   String lastEppUpdateClientId;
 
   /** The time when this resource was created. */
@@ -135,10 +142,16 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   @Transient
   ImmutableSortedMap<DateTime, Key<CommitLogManifest>> revisions = ImmutableSortedMap.of();
 
-  public final String getRepoId() {
+  public String getRepoId() {
     return repoId;
   }
 
+  // Hibernate needs this to populate the repo ID, but no one else should ever use it
+  @SuppressWarnings("UnusedMethod")
+  private void setRepoId(String repoId) {
+    this.repoId = repoId;
+  }
+
   public final DateTime getCreationTime() {
     return creationTime.getTimestamp();
   }
@@ -183,6 +196,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   /** Get the foreign key string for this resource. */
   public abstract String getForeignKey();
 
+  /** Create the VKey for the specified EPP resource. */
+  public abstract VKey<? extends EppResource> createVKey();
+
   /** Override of {@link Buildable#asBuilder} so that the extra methods are visible. */
   @Override
   public abstract Builder<?, ?> asBuilder();
@@ -191,8 +207,8 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   public interface ForeignKeyedEppResource {}
 
   /** An interface for resources that have transfer data. */
-  public interface ResourceWithTransferData {
-    TransferData getTransferData();
+  public interface ResourceWithTransferData<T extends TransferData> {
+    T getTransferData();
 
     /**
      * The time that this resource was last transferred.
@@ -203,15 +219,16 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   }
 
   /** An interface for builders of resources that have transfer data. */
-  public interface BuilderWithTransferData<B extends BuilderWithTransferData<B>> {
-    B setTransferData(TransferData transferData);
+  public interface BuilderWithTransferData<
+      T extends TransferData, B extends BuilderWithTransferData<T, B>> {
+    B setTransferData(T transferData);
 
     /** Set the time when this resource was transferred. */
     B setLastTransferTime(DateTime lastTransferTime);
   }
 
   /** Abstract builder for {@link EppResource} types. */
-  public abstract static class Builder<T extends EppResource, B extends Builder<?, ?>>
+  public abstract static class Builder<T extends EppResource, B extends Builder<T, B>>
       extends GenericBuilder<T, B> {
 
     /** Create a {@link Builder} wrapping a new instance. */
@@ -329,18 +346,18 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     }
   }
 
-  static final CacheLoader<Key<? extends EppResource>, EppResource> CACHE_LOADER =
-      new CacheLoader<Key<? extends EppResource>, EppResource>() {
+  static final CacheLoader<VKey<? extends EppResource>, EppResource> CACHE_LOADER =
+      new CacheLoader<VKey<? extends EppResource>, EppResource>() {
 
         @Override
-        public EppResource load(Key<? extends EppResource> key) {
-          return tm().doTransactionless(() -> ofy().load().key(key).now());
+        public EppResource load(VKey<? extends EppResource> key) {
+          return tm().doTransactionless(() -> tm().load(key));
         }
 
         @Override
-        public Map<Key<? extends EppResource>, EppResource> loadAll(
-            Iterable<? extends Key<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> loadMultiple(keys));
+        public Map<VKey<? extends EppResource>, EppResource> loadAll(
+            Iterable<? extends VKey<? extends EppResource>> keys) {
+          return tm().doTransactionless(() -> loadAsMap(keys));
         }
       };
 
@@ -352,10 +369,10 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * directly should of course never use the cache.
    */
   @NonFinalForTesting
-  private static LoadingCache<Key<? extends EppResource>, EppResource> cacheEppResources =
+  private static LoadingCache<VKey<? extends EppResource>, EppResource> cacheEppResources =
       createEppResourcesCache(getEppResourceCachingDuration());
 
-  private static LoadingCache<Key<? extends EppResource>, EppResource> createEppResourcesCache(
+  private static LoadingCache<VKey<? extends EppResource>, EppResource> createEppResourcesCache(
       Duration expiry) {
     return CacheBuilder.newBuilder()
         .expireAfterWrite(expiry.getMillis(), MILLISECONDS)
@@ -369,29 +386,16 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     cacheEppResources = createEppResourcesCache(effectiveExpiry);
   }
 
-  private static ImmutableMap<Key<? extends EppResource>, EppResource> loadMultiple(
-      Iterable<? extends Key<? extends EppResource>> keys) {
-    // This cast is safe because, in Objectify, Key<? extends EppResource> can also be
-    // treated as a Key<EppResource>.
-    @SuppressWarnings("unchecked")
-    ImmutableList<Key<EppResource>> typedKeys =
-        Streams.stream(keys).map(key -> (Key<EppResource>) key).collect(toImmutableList());
-
-    // Typing shenanigans are required to return the map with the correct required generic type.
-    return ofy().load().keys(typedKeys).entrySet().stream()
-        .collect(ImmutableMap.toImmutableMap(Entry::getKey, Entry::getValue));
-  }
-
   /**
    * Loads the given EppResources by their keys using the cache (if enabled).
    *
    * <p>Don't use this unless you really need it for performance reasons, and be sure that you are
    * OK with the trade-offs in loss of transactional consistency.
    */
-  public static ImmutableMap<Key<? extends EppResource>, EppResource> loadCached(
-      Iterable<Key<? extends EppResource>> keys) {
+  public static ImmutableMap<VKey<? extends EppResource>, EppResource> loadCached(
+      Iterable<VKey<? extends EppResource>> keys) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return loadMultiple(keys);
+      return loadAsMap(keys);
     }
     try {
       return cacheEppResources.getAll(keys);
@@ -406,9 +410,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>Don't use this unless you really need it for performance reasons, and be sure that you are
    * OK with the trade-offs in loss of transactional consistency.
    */
-  public static <T extends EppResource> T loadCached(Key<T> key) {
+  public static <T extends EppResource> T loadCached(VKey<T> key) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return ofy().load().key(key).now();
+      return tm().load(key);
     }
     try {
       // Safe to cast because loading a Key<T> returns an entity of type T.
@@ -419,4 +423,17 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
       throw new RuntimeException("Error loading cached EppResources", e.getCause());
     }
   }
+
+  private static ImmutableMap<VKey<? extends EppResource>, EppResource> loadAsMap(
+      Iterable<? extends VKey<? extends EppResource>> keys) {
+    return StreamSupport.stream(keys.spliterator(), false)
+        // It's possible for us to receive the same key more than once which causes
+        // the immutable map build to break with a duplicate key, so we have to ensure key
+        // uniqueness.
+        .distinct()
+        // We have to use "key -> key" here instead of the identity() function, because
+        // the latter breaks the fairly complicated generic type checking required by the
+        // caching interface.
+        .collect(toImmutableMap(key -> key, key -> tm().load(key)));
+  }
 }

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -17,6 +17,7 @@ package google.registry.model;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;
@@ -38,6 +39,7 @@ import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
 import google.registry.model.registry.Registry;
+import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferStatus;
 import java.util.List;
@@ -141,7 +143,7 @@ public final class EppResourceUtils {
     T resource =
         useCache
             ? EppResource.loadCached(fki.getResourceKey())
-            : ofy().load().key(fki.getResourceKey()).now();
+            : tm().maybeLoad(fki.getResourceKey()).orElse(null);
     if (resource == null || isAtOrAfter(now, resource.getDeletionTime())) {
       return Optional.empty();
     }
@@ -221,17 +223,23 @@ public final class EppResourceUtils {
   }
 
   /** Process an automatic transfer on a resource. */
-  public static <B extends EppResource.Builder<?, B> & BuilderWithTransferData<B>>
+  public static <
+          T extends TransferData,
+          B extends EppResource.Builder<?, B> & BuilderWithTransferData<T, B>>
       void setAutomaticTransferSuccessProperties(B builder, TransferData transferData) {
     checkArgument(TransferStatus.PENDING.equals(transferData.getTransferStatus()));
-    builder.removeStatusValue(StatusValue.PENDING_TRANSFER)
-        .setTransferData(transferData.asBuilder()
-            .setTransferStatus(TransferStatus.SERVER_APPROVED)
-            .setServerApproveEntities(null)
-            .setServerApproveBillingEvent(null)
-            .setServerApproveAutorenewEvent(null)
-            .setServerApproveAutorenewPollMessage(null)
-            .build())
+    TransferData.Builder transferDataBuilder = transferData.asBuilder();
+    transferDataBuilder.setTransferStatus(TransferStatus.SERVER_APPROVED);
+    transferDataBuilder.setServerApproveEntities(null);
+    if (transferData instanceof DomainTransferData) {
+      ((DomainTransferData.Builder) transferDataBuilder)
+          .setServerApproveBillingEvent(null)
+          .setServerApproveAutorenewEvent(null)
+          .setServerApproveAutorenewPollMessage(null);
+    }
+    builder
+        .removeStatusValue(StatusValue.PENDING_TRANSFER)
+        .setTransferData((T) transferDataBuilder.build())
         .setLastTransferTime(transferData.getPendingTransferExpirationTime())
         .setPersistedCurrentSponsorClientId(transferData.getGainingClientId());
   }
@@ -244,10 +252,11 @@ public final class EppResourceUtils {
    * </ul>
    */
   public static <
-          T extends EppResource & ResourceWithTransferData,
-          B extends EppResource.Builder<?, B> & BuilderWithTransferData<B>>
-      void projectResourceOntoBuilderAtTime(T resource, B builder, DateTime now) {
-    TransferData transferData = resource.getTransferData();
+          T extends TransferData,
+          E extends EppResource & ResourceWithTransferData<T>,
+          B extends EppResource.Builder<?, B> & BuilderWithTransferData<T, B>>
+      void projectResourceOntoBuilderAtTime(E resource, B builder, DateTime now) {
+    T transferData = resource.getTransferData();
     // If there's a pending transfer that has expired, process it.
     DateTime expirationTime = transferData.getPendingTransferExpirationTime();
     if (TransferStatus.PENDING.equals(transferData.getTransferStatus())

core/src/main/java/google/registry/model/ModelUtils.java

@@ -34,6 +34,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Parent;
+import google.registry.persistence.VKey;
 import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
@@ -139,10 +140,13 @@ public class ModelUtils {
 
       // If the field's type is the same as the field's class object, then it's a non-parameterized
       // type, and thus we just add it directly. We also don't bother looking at the parameterized
-      // types of Key objects, since they are just references to other objects and don't actually
-      // embed themselves in the persisted object anyway.
+      // types of Key and VKey objects, since they are just references to other objects and don't
+      // actually embed themselves in the persisted object anyway.
       Class<?> fieldClazz = field.getType();
       Type fieldType = field.getGenericType();
+      if (VKey.class.equals(fieldClazz)) {
+        continue;
+      }
       builder.add(fieldClazz);
       if (fieldType.equals(fieldClazz) || Key.class.equals(clazz)) {
         continue;

core/src/main/java/google/registry/model/OteStats.java

@@ -50,7 +50,8 @@ import java.util.stream.Collectors;
 public class OteStats {
 
   /**
-   * Returns the statistics about the OT&E actions that have been taken by a particular registrar.
+   * Returns the statistics about the OT&E actions that have been taken by a particular
+   * registrar.
    */
   public static OteStats getFromRegistrar(String registrarName) {
     return new OteStats().recordRegistrarHistory(registrarName);

core/src/main/java/google/registry/model/ResourceTransferUtils.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -35,6 +36,7 @@ import google.registry.model.poll.PendingActionNotificationResponse.ContactPendi
 import google.registry.model.poll.PendingActionNotificationResponse.DomainPendingActionNotificationResponse;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferResponse.ContactTransferResponse;
@@ -62,12 +64,13 @@ public final class ResourceTransferUtils {
     if (eppResource instanceof ContactResource) {
       builder = new ContactTransferResponse.Builder().setContactId(eppResource.getForeignKey());
     } else {
+      DomainTransferData domainTransferData = (DomainTransferData) transferData;
       builder =
           new DomainTransferResponse.Builder()
               .setFullyQualifiedDomainName(eppResource.getForeignKey())
               .setExtendedRegistrationExpirationTime(
-                  ADD_EXDATE_STATUSES.contains(transferData.getTransferStatus())
-                      ? transferData.getTransferredRegistrationExpirationTime()
+                  ADD_EXDATE_STATUSES.contains(domainTransferData.getTransferStatus())
+                      ? domainTransferData.getTransferredRegistrationExpirationTime()
                       : null);
     }
     builder.setGainingClientId(transferData.getGainingClientId())
@@ -114,7 +117,7 @@ public final class ResourceTransferUtils {
             R resource, R newResource, DateTime now, HistoryEntry historyEntry) {
     if (resource.getStatusValues().contains(StatusValue.PENDING_TRANSFER)) {
       TransferData oldTransferData = resource.getTransferData();
-      ofy().delete().keys(oldTransferData.getServerApproveEntities());
+      tm().delete(oldTransferData.getServerApproveEntities());
       ofy()
           .save()
           .entity(
@@ -141,23 +144,25 @@ public final class ResourceTransferUtils {
    */
   private static <
           R extends EppResource & ResourceWithTransferData,
-          B extends EppResource.Builder<R, B> & BuilderWithTransferData<B>>
+          B extends EppResource.Builder<R, B> & BuilderWithTransferData<TransferData, B>>
       B resolvePendingTransfer(R resource, TransferStatus transferStatus, DateTime now) {
     checkArgument(
         resource.getStatusValues().contains(StatusValue.PENDING_TRANSFER),
         "Resource is not in pending transfer status.");
-    checkArgument(
-        !TransferData.EMPTY.equals(resource.getTransferData()),
-        "No old transfer data to resolve.");
+    checkArgument(!resource.getTransferData().isEmpty(), "No old transfer data to resolve.");
     @SuppressWarnings("unchecked")
     B builder = (B) resource.asBuilder();
+
     return builder
         .removeStatusValue(StatusValue.PENDING_TRANSFER)
         .setTransferData(
-            resource.getTransferData().copyConstantFieldsToBuilder()
-                .setTransferStatus(transferStatus)
-                .setPendingTransferExpirationTime(checkNotNull(now))
-                .build());
+            (TransferData)
+                resource
+                    .getTransferData()
+                    .copyConstantFieldsToBuilder()
+                    .setTransferStatus(transferStatus)
+                    .setPendingTransferExpirationTime(checkNotNull(now))
+                    .build());
   }
 
   /**
@@ -170,7 +175,7 @@ public final class ResourceTransferUtils {
    */
   public static <
           R extends EppResource & ResourceWithTransferData,
-          B extends EppResource.Builder<R, B> & BuilderWithTransferData<B>>
+          B extends EppResource.Builder<R, B> & BuilderWithTransferData<TransferData, B>>
       R approvePendingTransfer(R resource, TransferStatus transferStatus, DateTime now) {
     checkArgument(transferStatus.isApproved(), "Not an approval transfer status");
     B builder = resolvePendingTransfer(resource, transferStatus, now);

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -30,7 +30,7 @@ public class UpdateAutoTimestamp extends ImmutableObject {
 
   DateTime timestamp;
 
-  /** Returns the timestamp, or {@link START_OF_TIME} if it's null. */
+  /** Returns the timestamp, or {@code START_OF_TIME} if it's null. */
   public DateTime getTimestamp() {
     return Optional.ofNullable(timestamp).orElse(START_OF_TIME);
   }

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -30,6 +30,7 @@ import com.google.common.collect.Sets;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
 import com.googlecode.objectify.annotation.Parent;
@@ -43,14 +44,28 @@ import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithLongVKey;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embedded;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.Transient;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
 /** A billable event in a domain's lifecycle. */
+@MappedSuperclass
+@WithLongVKey
 public abstract class BillingEvent extends ImmutableObject
     implements Buildable, TransferServerApproveEntity {
 
@@ -93,24 +108,41 @@ public abstract class BillingEvent extends ImmutableObject
 
   /** Entity id. */
   @Id
-  long id;
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  Long id;
 
-  @Parent
-  @DoNotHydrate
-  Key<HistoryEntry> parent;
+  @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
 
   /** The registrar to bill. */
   @Index
+  @Column(name = "registrarId", nullable = false)
   String clientId;
 
+  /** Revision id of the entry in DomainHistory table that ths bill belongs to. */
+  // TODO(shicong): Add foreign key constraint when DomainHistory table is generated
+  @Ignore
+  @Column(nullable = false)
+  Long domainHistoryRevisionId;
+
+  /** ID of the EPP resource that the bill is for. */
+  // TODO(shicong): Add foreign key constraint when we expand DatastoreHelp for Postgresql
+  @Ignore
+  @Column(nullable = false)
+  String domainRepoId;
+
   /** When this event was created. For recurring events, this is also the recurrence start time. */
   @Index
+  @Column(nullable = false)
   DateTime eventTime;
 
   /** The reason for the bill. */
+  @Enumerated(EnumType.STRING)
+  @Column(nullable = false)
   Reason reason;
 
   /** The fully qualified domain name of the domain that the bill is for. */
+  @Column(name = "domain_name", nullable = false)
   String targetId;
 
   @Nullable
@@ -120,6 +152,14 @@ public abstract class BillingEvent extends ImmutableObject
     return clientId;
   }
 
+  public long getDomainHistoryRevisionId() {
+    return domainHistoryRevisionId;
+  }
+
+  public String getDomainRepoId() {
+    return domainRepoId;
+  }
+
   public DateTime getEventTime() {
     return eventTime;
   }
@@ -163,7 +203,7 @@ public abstract class BillingEvent extends ImmutableObject
       return thisCastToDerived();
     }
 
-    public B setId(Long id) {
+    public B setId(long id) {
       getInstance().id = id;
       return thisCastToDerived();
     }
@@ -173,6 +213,16 @@ public abstract class BillingEvent extends ImmutableObject
       return thisCastToDerived();
     }
 
+    public B setDomainHistoryRevisionId(long domainHistoryRevisionId) {
+      getInstance().domainHistoryRevisionId = domainHistoryRevisionId;
+      return thisCastToDerived();
+    }
+
+    public B setDomainRepoId(String domainRepoId) {
+      getInstance().domainRepoId = domainRepoId;
+      return thisCastToDerived();
+    }
+
     public B setEventTime(DateTime eventTime) {
       getInstance().eventTime = eventTime;
       return thisCastToDerived();
@@ -194,6 +244,7 @@ public abstract class BillingEvent extends ImmutableObject
     }
 
     public B setParent(Key<HistoryEntry> parentKey) {
+      // TODO(shicong): Figure out how to set domainHistoryRevisionId and domainRepoId
       getInstance().parent = parentKey;
       return thisCastToDerived();
     }
@@ -213,9 +264,23 @@ public abstract class BillingEvent extends ImmutableObject
   /** A one-time billable event. */
   @ReportedOn
   @Entity
+  @javax.persistence.Entity(name = "BillingEvent")
+  @javax.persistence.Table(
+      indexes = {
+        @javax.persistence.Index(columnList = "registrarId"),
+        @javax.persistence.Index(columnList = "eventTime"),
+        @javax.persistence.Index(columnList = "billingTime"),
+        @javax.persistence.Index(columnList = "syntheticCreationTime"),
+        @javax.persistence.Index(columnList = "allocation_token_id")
+      })
+  @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
   public static class OneTime extends BillingEvent {
 
     /** The billable value. */
+    @AttributeOverrides({
+      @AttributeOverride(name = "money.amount", column = @Column(name = "cost_amount")),
+      @AttributeOverride(name = "money.currency", column = @Column(name = "cost_currency"))
+    })
     Money cost;
 
     /** When the cost should be billed. */
@@ -223,8 +288,8 @@ public abstract class BillingEvent extends ImmutableObject
     DateTime billingTime;
 
     /**
-     * The period in years of the action being billed for, if applicable, otherwise null.
-     * Used for financial reporting.
+     * The period in years of the action being billed for, if applicable, otherwise null. Used for
+     * financial reporting.
      */
     @IgnoreSave(IfNull.class)
     Integer periodYears = null;
@@ -240,15 +305,21 @@ public abstract class BillingEvent extends ImmutableObject
 
     /**
      * For {@link Flag#SYNTHETIC} events, a {@link Key} to the {@link BillingEvent} from which this
-     * OneTime was created. This is needed in order to properly match billing events against
-     * {@link Cancellation}s.
+     * OneTime was created. This is needed in order to properly match billing events against {@link
+     * Cancellation}s.
      */
-    Key<? extends BillingEvent> cancellationMatchingBillingEvent;
+    @Column(name = "cancellation_matching_billing_recurrence_id")
+    VKey<? extends BillingEvent> cancellationMatchingBillingEvent;
 
     /**
      * The {@link AllocationToken} used in the creation of this event, or null if one was not used.
+     *
+     * <p>TODO(shicong): Add foreign key constraint when AllocationToken schema is generated
      */
-    @Index @Nullable Key<AllocationToken> allocationToken;
+    @Column(name = "allocation_token_id")
+    @Index
+    @Nullable
+    VKey<AllocationToken> allocationToken;
 
     public Money getCost() {
       return cost;
@@ -266,14 +337,19 @@ public abstract class BillingEvent extends ImmutableObject
       return syntheticCreationTime;
     }
 
-    public Key<? extends BillingEvent> getCancellationMatchingBillingEvent() {
+    public VKey<? extends BillingEvent> getCancellationMatchingBillingEvent() {
       return cancellationMatchingBillingEvent;
     }
 
-    public Optional<Key<AllocationToken>> getAllocationToken() {
+    public Optional<VKey<AllocationToken>> getAllocationToken() {
       return Optional.ofNullable(allocationToken);
     }
 
+    @Override
+    public VKey<OneTime> createVKey() {
+      return VKey.createOfy(getClass(), Key.create(this));
+    }
+
     @Override
     public Builder asBuilder() {
       return new Builder(clone(this));
@@ -311,12 +387,12 @@ public abstract class BillingEvent extends ImmutableObject
       }
 
       public Builder setCancellationMatchingBillingEvent(
-          Key<? extends BillingEvent> cancellationMatchingBillingEvent) {
+          VKey<? extends BillingEvent> cancellationMatchingBillingEvent) {
         getInstance().cancellationMatchingBillingEvent = cancellationMatchingBillingEvent;
         return this;
       }
 
-      public Builder setAllocationToken(@Nullable Key<AllocationToken> allocationToken) {
+      public Builder setAllocationToken(@Nullable VKey<AllocationToken> allocationToken) {
         getInstance().allocationToken = allocationToken;
         return this;
       }
@@ -361,6 +437,15 @@ public abstract class BillingEvent extends ImmutableObject
    */
   @ReportedOn
   @Entity
+  @javax.persistence.Entity(name = "BillingRecurrence")
+  @javax.persistence.Table(
+      indexes = {
+        @javax.persistence.Index(columnList = "registrarId"),
+        @javax.persistence.Index(columnList = "eventTime"),
+        @javax.persistence.Index(columnList = "recurrenceEndTime"),
+        @javax.persistence.Index(columnList = "recurrence_time_of_year")
+      })
+  @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
   public static class Recurring extends BillingEvent {
 
     /**
@@ -384,6 +469,10 @@ public abstract class BillingEvent extends ImmutableObject
      * model, whereas the billing time is a fixed {@link org.joda.time.Duration} later.
      */
     @Index
+    @Embedded
+    @AttributeOverrides({
+      @AttributeOverride(name = "timeString", column = @Column(name = "recurrence_time_of_year"))
+    })
     TimeOfYear recurrenceTimeOfYear;
 
     public DateTime getRecurrenceEndTime() {
@@ -394,6 +483,11 @@ public abstract class BillingEvent extends ImmutableObject
       return recurrenceTimeOfYear;
     }
 
+    @Override
+    public VKey<Recurring> createVKey() {
+      return VKey.createOfy(getClass(), Key.create(this));
+    }
+
     @Override
     public Builder asBuilder() {
       return new Builder(clone(this));
@@ -434,6 +528,14 @@ public abstract class BillingEvent extends ImmutableObject
    */
   @ReportedOn
   @Entity
+  @javax.persistence.Entity(name = "BillingCancellation")
+  @javax.persistence.Table(
+      indexes = {
+        @javax.persistence.Index(columnList = "registrarId"),
+        @javax.persistence.Index(columnList = "eventTime"),
+        @javax.persistence.Index(columnList = "billingTime")
+      })
+  @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
   public static class Cancellation extends BillingEvent {
 
     /** The billing time of the charge that is being cancelled. */
@@ -446,7 +548,8 @@ public abstract class BillingEvent extends ImmutableObject
      * <p>Although the type is {@link Key} the name "ref" is preserved for historical reasons.
      */
     @IgnoreSave(IfNull.class)
-    Key<BillingEvent.OneTime> refOneTime = null;
+    @Column(name = "billing_event_id")
+    VKey<BillingEvent.OneTime> refOneTime = null;
 
     /**
      * The recurring billing event to cancel, or null for non-autorenew cancellations.
@@ -454,13 +557,14 @@ public abstract class BillingEvent extends ImmutableObject
      * <p>Although the type is {@link Key} the name "ref" is preserved for historical reasons.
      */
     @IgnoreSave(IfNull.class)
-    Key<BillingEvent.Recurring> refRecurring = null;
+    @Column(name = "billing_recurrence_id")
+    VKey<BillingEvent.Recurring> refRecurring = null;
 
     public DateTime getBillingTime() {
       return billingTime;
     }
 
-    public Key<? extends BillingEvent> getEventKey() {
+    public VKey<? extends BillingEvent> getEventKey() {
       return firstNonNull(refOneTime, refRecurring);
     }
 
@@ -492,13 +596,20 @@ public abstract class BillingEvent extends ImmutableObject
           .setParent(historyEntry);
       // Set the grace period's billing event using the appropriate Cancellation builder method.
       if (gracePeriod.getOneTimeBillingEvent() != null) {
-        builder.setOneTimeEventKey(gracePeriod.getOneTimeBillingEvent());
+        builder.setOneTimeEventKey(
+            VKey.createOfy(BillingEvent.OneTime.class, gracePeriod.getOneTimeBillingEvent()));
       } else if (gracePeriod.getRecurringBillingEvent() != null) {
-        builder.setRecurringEventKey(gracePeriod.getRecurringBillingEvent());
+        builder.setRecurringEventKey(
+            VKey.createOfy(BillingEvent.Recurring.class, gracePeriod.getRecurringBillingEvent()));
       }
       return builder.build();
     }
 
+    @Override
+    public VKey<Cancellation> createVKey() {
+      return VKey.createOfy(getClass(), Key.create(this));
+    }
+
     @Override
     public Builder asBuilder() {
       return new Builder(clone(this));
@@ -518,12 +629,12 @@ public abstract class BillingEvent extends ImmutableObject
         return this;
       }
 
-      public Builder setOneTimeEventKey(Key<BillingEvent.OneTime> eventKey) {
+      public Builder setOneTimeEventKey(VKey<BillingEvent.OneTime> eventKey) {
         getInstance().refOneTime = eventKey;
         return this;
       }
 
-      public Builder setRecurringEventKey(Key<BillingEvent.Recurring> eventKey) {
+      public Builder setRecurringEventKey(VKey<BillingEvent.Recurring> eventKey) {
         getInstance().refRecurring = eventKey;
         return this;
       }
@@ -540,9 +651,7 @@ public abstract class BillingEvent extends ImmutableObject
     }
   }
 
-  /**
-   * An event representing a modification of an existing one-time billing event.
-   */
+  /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
   public static class Modification extends BillingEvent {
@@ -576,6 +685,11 @@ public abstract class BillingEvent extends ImmutableObject
       return new Builder(clone(this));
     }
 
+    @Override
+    public VKey<Modification> createVKey() {
+      return VKey.createOfy(this.getClass(), Key.create(this));
+    }
+
     /**
      * Create a new Modification billing event which is a refund of the given OneTime billing event
      * and that is parented off the given HistoryEntry.

core/src/main/java/google/registry/model/common/Cursor.java

@@ -21,6 +21,7 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -28,16 +29,18 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.registry.Registry;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.List;
 import org.joda.time.DateTime;
 
 /**
- * Shared entity for date cursors. This type supports both "scoped" cursors (i.e. per resource
- * of a given type, such as a TLD) and global (i.e. one per environment) cursors, defined internally
- * as scoped on {@link EntityGroupRoot}.
+ * Shared entity for date cursors. This type supports both "scoped" cursors (i.e. per resource of a
+ * given type, such as a TLD) and global (i.e. one per environment) cursors, defined internally as
+ * scoped on {@link EntityGroupRoot}.
  */
 @Entity
-public class Cursor extends ImmutableObject {
+public class Cursor extends ImmutableObject implements DatastoreEntity {
 
   /** The types of cursors, used as the string id field for each cursor in Datastore. */
   public enum CursorType {
@@ -134,6 +137,11 @@ public class Cursor extends ImmutableObject {
     return CursorType.valueOf(String.join("_", id.subList(1, id.size())));
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // Cursors are not converted since they are ephemeral
+  }
+
   /**
    * Checks that the type of the scoped object (or null) matches the required type for the specified
    * cursor (or null, if the cursor is a global cursor).

core/src/main/java/google/registry/model/common/EntityGroupRoot.java

@@ -14,27 +14,30 @@
 
 package google.registry.model.common;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.BackupGroupRoot;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /**
  * The root key for the entity group which is known as the cross-tld entity group for historical
  * reasons.
  *
  * <p>This exists as a storage place for common configuration options and global settings that
- * aren't updated too frequently.  Entities in this entity group are usually cached upon load.  The
+ * aren't updated too frequently. Entities in this entity group are usually cached upon load. The
  * reason this common entity group exists is because it enables strongly consistent queries and
- * updates across this seldomly updated data.  This shared entity group also helps cut down on
- * a potential ballooning in the number of entity groups enlisted in transactions.
+ * updates across this seldomly updated data. This shared entity group also helps cut down on a
+ * potential ballooning in the number of entity groups enlisted in transactions.
  *
- * <p>Historically, each TLD used to have a separate namespace, and all entities for a TLD were in
- * a single EntityGroupRoot for that TLD.  Hence why there was a "cross-tld" entity group -- it was
- * the entity group for the single namespace where global data applicable for all TLDs lived.
+ * <p>Historically, each TLD used to have a separate namespace, and all entities for a TLD were in a
+ * single EntityGroupRoot for that TLD. Hence why there was a "cross-tld" entity group -- it was the
+ * entity group for the single namespace where global data applicable for all TLDs lived.
  */
 @Entity
-public class EntityGroupRoot extends BackupGroupRoot {
+public class EntityGroupRoot extends BackupGroupRoot implements DatastoreEntity {
 
   @SuppressWarnings("unused")
   @Id
@@ -44,4 +47,9 @@ public class EntityGroupRoot extends BackupGroupRoot {
   public static Key<EntityGroupRoot> getCrossTldKey() {
     return Key.create(EntityGroupRoot.class, "cross-tld");
   }
+
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
 }

core/src/main/java/google/registry/model/common/TimeOfYear.java

@@ -29,20 +29,22 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
 import java.util.List;
+import javax.persistence.Embeddable;
 import org.joda.time.DateTime;
 
 /**
  * A time of year (month, day, millis of day) that can be stored in a sort-friendly format.
  *
- * <p>This is conceptually similar to {@code MonthDay} in Joda or more generally to Joda's
- * {@code Partial}, but the parts we need are too simple to justify a full implementation of
- * {@code Partial}.
+ * <p>This is conceptually similar to {@code MonthDay} in Joda or more generally to Joda's {@code
+ * Partial}, but the parts we need are too simple to justify a full implementation of {@code
+ * Partial}.
  *
  * <p>For simplicity, the native representation of this class's data is its stored format. This
  * allows it to be embeddable with no translation needed and also delays parsing of the string on
  * load until it's actually needed.
  */
 @Embed
+@Embeddable
 public class TimeOfYear extends ImmutableObject {
 
   /**

core/src/main/java/google/registry/model/contact/ContactCommand.java

@@ -37,7 +37,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 /** A collection of {@link ContactResource} commands. */
 public class ContactCommand {
 
-  /** The fields on "chgType" from {@link "http://tools.ietf.org/html/rfc5733"}. */
+  /** The fields on "chgType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
   @XmlTransient
   public static class ContactCreateOrChange extends ImmutableObject
       implements ResourceCreateOrChange<ContactResource.Builder> {
@@ -111,8 +111,8 @@ public class ContactCommand {
   }
 
   /**
-   * A create command for a {@link ContactResource}, mapping "createType" from
-   * {@link "http://tools.ietf.org/html/rfc5733"}.
+   * A create command for a {@link ContactResource}, mapping "createType" from <a
+   * href="http://tools.ietf.org/html/rfc5733">RFC5733</a>}.
    */
   @XmlType(propOrder = {"contactId", "postalInfo", "voice", "fax", "email", "authInfo", "disclose"})
   @XmlRootElement

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -19,6 +19,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
 
 import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
@@ -29,15 +30,19 @@ import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.PostalInfo.Type;
-import google.registry.model.transfer.TransferData;
+import google.registry.model.transfer.ContactTransferData;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Stream;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Embedded;
-import javax.persistence.Transient;
 import javax.xml.bind.annotation.XmlElement;
 import org.joda.time.DateTime;
 
@@ -53,14 +58,16 @@ import org.joda.time.DateTime;
     name = "Contact",
     indexes = {
       @javax.persistence.Index(columnList = "creationTime"),
-      @javax.persistence.Index(columnList = "currentSponsorClientId"),
+      @javax.persistence.Index(columnList = "currentSponsorRegistrarId"),
       @javax.persistence.Index(columnList = "deletionTime"),
       @javax.persistence.Index(columnList = "contactId", unique = true),
       @javax.persistence.Index(columnList = "searchName")
     })
 @ExternalMessagingName("contact")
+@WithStringVKey
+@Access(AccessType.FIELD)
 public class ContactResource extends EppResource
-    implements ForeignKeyedEppResource, ResourceWithTransferData {
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource, ResourceWithTransferData {
 
   /**
    * Unique identifier for this contact.
@@ -166,8 +173,7 @@ public class ContactResource extends EppResource
   ContactAuthInfo authInfo;
 
   /** Data about any pending or past transfers on this contact. */
-  // TODO(b/153363295): Figure out how to persist transfer data
-  @Transient TransferData transferData;
+  ContactTransferData transferData;
 
   /**
    * The time that this resource was last transferred.
@@ -192,6 +198,19 @@ public class ContactResource extends EppResource
   })
   Disclose disclose;
 
+  @Override
+  public VKey<ContactResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.createOfy(ContactResource.class, Key.create(this));
+  }
+
+  @Override
+  @javax.persistence.Id
+  @Access(AccessType.PROPERTY)
+  public String getRepoId() {
+    return super.getRepoId();
+  }
+
   public String getContactId() {
     return contactId;
   }
@@ -233,8 +252,8 @@ public class ContactResource extends EppResource
   }
 
   @Override
-  public final TransferData getTransferData() {
-    return Optional.ofNullable(transferData).orElse(TransferData.EMPTY);
+  public final ContactTransferData getTransferData() {
+    return Optional.ofNullable(transferData).orElse(ContactTransferData.EMPTY);
   }
 
   @Override
@@ -276,7 +295,7 @@ public class ContactResource extends EppResource
 
   /** A builder for constructing {@link ContactResource}, since it is immutable. */
   public static class Builder extends EppResource.Builder<ContactResource, Builder>
-      implements BuilderWithTransferData<Builder> {
+      implements BuilderWithTransferData<ContactTransferData, Builder> {
 
     public Builder() {}
 
@@ -341,7 +360,7 @@ public class ContactResource extends EppResource
     }
 
     @Override
-    public Builder setTransferData(TransferData transferData) {
+    public Builder setTransferData(ContactTransferData transferData) {
       getInstance().transferData = transferData;
       return this;
     }
@@ -371,7 +390,7 @@ public class ContactResource extends EppResource
     public ContactResource build() {
       ContactResource instance = getInstance();
       // If TransferData is totally empty, set it to null.
-      if (TransferData.EMPTY.equals(instance.transferData)) {
+      if (ContactTransferData.EMPTY.equals(instance.transferData)) {
         setTransferData(null);
       }
       // Set the searchName using the internationalized and localized postal info names.

core/src/main/java/google/registry/model/contact/Disclose.java

@@ -27,7 +27,7 @@ import javax.persistence.Embedded;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlType;
 
-/** The "discloseType" from {@link "http://tools.ietf.org/html/rfc5733"}. */
+/** The "discloseType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
 @Embed
 @Embeddable
 @XmlType(propOrder = {"name", "org", "addr", "voice", "fax", "email"})
@@ -76,7 +76,7 @@ public class Disclose extends ImmutableObject {
     return flag;
   }
 
-  /** The "intLocType" from {@link "http://tools.ietf.org/html/rfc5733"}. */
+  /** The "intLocType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
   @Embed
   public static class PostalInfoChoice extends ImmutableObject {
     @XmlAttribute

core/src/main/java/google/registry/model/contact/PostalInfo.java

@@ -32,8 +32,8 @@ import javax.xml.bind.annotation.adapters.NormalizedStringAdapter;
 import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 /**
- * Implementation of both "postalInfoType" and "chgPostalInfoType" from {@link
- * "http://tools.ietf.org/html/rfc5733"}.
+ * Implementation of both "postalInfoType" and "chgPostalInfoType" from <a href=
+ * "http://tools.ietf.org/html/rfc5733">RFC5733</a>.
  */
 @Embed
 @Embeddable

core/src/main/java/google/registry/model/domain/DesignatedContact.java

@@ -18,9 +18,11 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
 import google.registry.model.contact.ContactResource;
+import google.registry.persistence.VKey;
 import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlEnumValue;
 
@@ -36,6 +38,9 @@ import javax.xml.bind.annotation.XmlEnumValue;
  * situation with hosts where client-side renames would make that data stale. However, we sometimes
  * rename contacts internally ourselves, and it's easier to use the same model for both cases.
  *
+ * <p>This entity type is not persisted in Cloud SQL. The different roles are represented as
+ * separate fields in the Domain table.
+ *
  * @see <a href="http://tools.ietf.org/html/rfc5731#section-2.2">RFC 5731 - EPP Domain Name Mapping
  *     - Contact and Client Identifiers</a>
  */
@@ -58,22 +63,28 @@ public class DesignatedContact extends ImmutableObject {
     REGISTRANT
   }
 
-  public static DesignatedContact create(Type type, Key<ContactResource> contact) {
+  public static DesignatedContact create(Type type, VKey<ContactResource> contact) {
     DesignatedContact instance = new DesignatedContact();
     instance.type = type;
-    instance.contact = checkArgumentNotNull(contact, "Must specify contact key");
+    instance.contactVKey = checkArgumentNotNull(contact, "Must specify contact key");
+    instance.contact = contact.maybeGetOfyKey().orElse(null);
     return instance;
   }
 
   Type type;
 
   @Index Key<ContactResource> contact;
+  @Ignore VKey<ContactResource> contactVKey;
 
   public Type getType() {
     return type;
   }
 
-  public Key<ContactResource> getContactKey() {
-    return contact;
+  public VKey<ContactResource> getContactKey() {
+    return contactVKey;
+  }
+
+  public DesignatedContact reconstitute() {
+    return create(type, VKey.createOfy(ContactResource.class, contact));
   }
 }

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -63,9 +63,10 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registry.Registry;
-import google.registry.model.transfer.TransferData;
+import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.util.CollectionUtils;
 import java.util.HashSet;
@@ -74,13 +75,15 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
 import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
-import javax.persistence.Convert;
 import javax.persistence.ElementCollection;
 import javax.persistence.Embedded;
 import javax.persistence.JoinTable;
+import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
@@ -101,14 +104,18 @@ import org.joda.time.Interval;
     name = "Domain",
     indexes = {
       @javax.persistence.Index(columnList = "creationTime"),
-      @javax.persistence.Index(columnList = "currentSponsorClientId"),
+      @javax.persistence.Index(columnList = "currentSponsorRegistrarId"),
       @javax.persistence.Index(columnList = "deletionTime"),
-      @javax.persistence.Index(columnList = "fullyQualifiedDomainName"),
+      @javax.persistence.Index(columnList = "domainName"),
       @javax.persistence.Index(columnList = "tld")
     })
+@WithStringVKey
 @ExternalMessagingName("domain")
+@Access(AccessType.FIELD)
 public class DomainBase extends EppResource
-    implements ForeignKeyedEppResource, ResourceWithTransferData, DatastoreAndSqlEntity {
+    implements DatastoreAndSqlEntity,
+        ForeignKeyedEppResource,
+        ResourceWithTransferData<DomainTransferData> {
 
   /** The max number of years that a domain can be registered for, as set by ICANN policy. */
   public static final int MAX_REGISTRATION_YEARS = 10;
@@ -130,25 +137,19 @@ public class DomainBase extends EppResource
    *
    * @invariant fullyQualifiedDomainName == fullyQualifiedDomainName.toLowerCase(Locale.ENGLISH)
    */
+  // TODO(b/158858642): Rename this to domainName when we are off Datastore
+  @Column(name = "domainName")
   @Index String fullyQualifiedDomainName;
 
   /** The top level domain this is under, dernormalized from {@link #fullyQualifiedDomainName}. */
   @Index
   String tld;
 
-  /**
-   * References to hosts that are the nameservers for the domain.
-   *
-   * <p>This is a legacy field: we have to preserve it because it is still persisted and indexed in
-   * the datastore, but all external references go through nsHostVKeys.
-   */
-  @Index @ElementCollection @Transient Set<Key<HostResource>> nsHosts;
-
-  @Ignore
+  /** References to hosts that are the nameservers for the domain. */
+  @Index
   @ElementCollection
   @JoinTable(name = "DomainHost")
-  @Convert(converter = HostResource.VKeyHostResourceConverter.class)
-  Set<VKey<HostResource>> nsHostVKeys;
+  Set<VKey<HostResource>> nsHosts;
 
   /**
    * The union of the contacts visible via {@link #getContacts} and {@link #getRegistrant}.
@@ -157,6 +158,17 @@ public class DomainBase extends EppResource
    */
   @Transient Set<DesignatedContact> allContacts;
 
+  /**
+   * Contacts as they are stored in cloud SQL.
+   *
+   * <p>This information is duplicated in allContacts, and must be kept in sync with it.
+   */
+  @Ignore VKey<ContactResource> adminContact;
+
+  @Ignore VKey<ContactResource> billingContact;
+  @Ignore VKey<ContactResource> techContact;
+  @Ignore VKey<ContactResource> registrantContact;
+
   /** Authorization info (aka transfer secret) of the domain. */
   @Embedded
   @AttributeOverrides({
@@ -248,7 +260,7 @@ public class DomainBase extends EppResource
   String smdId;
 
   /** Data about any pending or past transfers on this domain. */
-  @Transient TransferData transferData;
+  DomainTransferData transferData;
 
   /**
    * The time that this resource was last transferred.
@@ -259,10 +271,40 @@ public class DomainBase extends EppResource
 
   @OnLoad
   void load() {
-    nsHostVKeys =
-        nullToEmptyImmutableCopy(nsHosts).stream()
-            .map(hostKey -> VKey.createOfy(HostResource.class, hostKey))
-            .collect(toImmutableSet());
+    // Reconstitute all of the contacts so that they have VKeys.
+    allContacts =
+        allContacts.stream().map(contact -> contact.reconstitute()).collect(toImmutableSet());
+    setContactFields(allContacts, true);
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Reconstitute the contact list.
+    ImmutableSet.Builder<DesignatedContact> contactsBuilder =
+        new ImmutableSet.Builder<DesignatedContact>();
+
+    if (registrantContact != null) {
+      contactsBuilder.add(
+          DesignatedContact.create(DesignatedContact.Type.REGISTRANT, registrantContact));
+    }
+    if (billingContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.BILLING, billingContact));
+    }
+    if (techContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.TECH, techContact));
+    }
+    if (adminContact != null) {
+      contactsBuilder.add(DesignatedContact.create(DesignatedContact.Type.ADMIN, adminContact));
+    }
+
+    allContacts = contactsBuilder.build();
+  }
+
+  @Override
+  @javax.persistence.Id
+  @Access(AccessType.PROPERTY)
+  public String getRepoId() {
+    return super.getRepoId();
   }
 
   public ImmutableSet<String> getSubordinateHosts() {
@@ -294,8 +336,8 @@ public class DomainBase extends EppResource
   }
 
   @Override
-  public TransferData getTransferData() {
-    return Optional.ofNullable(transferData).orElse(TransferData.EMPTY);
+  public DomainTransferData getTransferData() {
+    return Optional.ofNullable(transferData).orElse(DomainTransferData.EMPTY);
   }
 
   @Override
@@ -308,7 +350,7 @@ public class DomainBase extends EppResource
     return fullyQualifiedDomainName;
   }
 
-  public String getFullyQualifiedDomainName() {
+  public String getDomainName() {
     return fullyQualifiedDomainName;
   }
 
@@ -325,9 +367,7 @@ public class DomainBase extends EppResource
   }
 
   public ImmutableSet<VKey<HostResource>> getNameservers() {
-    // Since nsHostVKeys gets initialized both from setNameservers() and the OnLoad method, this
-    // should always be valid.
-    return nullToEmptyImmutableCopy(nsHostVKeys);
+    return nullToEmptyImmutableCopy(nsHosts);
   }
 
   public final String getCurrentSponsorClientId() {
@@ -376,7 +416,7 @@ public class DomainBase extends EppResource
   @Override
   public DomainBase cloneProjectedAtTime(final DateTime now) {
 
-    TransferData transferData = getTransferData();
+    DomainTransferData transferData = getTransferData();
     DateTime transferExpirationTime = transferData.getPendingTransferExpirationTime();
 
     // If there's a pending transfer that has expired, handle it.
@@ -412,8 +452,14 @@ public class DomainBase extends EppResource
               .setRegistrationExpirationTime(expirationDate)
               // Set the speculatively-written new autorenew events as the domain's autorenew
               // events.
-              .setAutorenewBillingEvent(transferData.getServerApproveAutorenewEvent())
-              .setAutorenewPollMessage(transferData.getServerApproveAutorenewPollMessage());
+              .setAutorenewBillingEvent(
+                  transferData.getServerApproveAutorenewEvent() == null
+                      ? null
+                      : transferData.getServerApproveAutorenewEvent().getOfyKey())
+              .setAutorenewPollMessage(
+                  transferData.getServerApproveAutorenewPollMessage() == null
+                      ? null
+                      : transferData.getServerApproveAutorenewPollMessage().getOfyKey());
       if (transferData.getTransferPeriod().getValue() == 1) {
         // Set the grace period using a key to the prescheduled transfer billing event.  Not using
         // GracePeriod.forBillingEvent() here in order to avoid the actual Datastore fetch.
@@ -424,7 +470,9 @@ public class DomainBase extends EppResource
                     transferExpirationTime.plus(
                         Registry.get(getTld()).getTransferGracePeriodLength()),
                     transferData.getGainingClientId(),
-                    transferData.getServerApproveBillingEvent())));
+                    transferData.getServerApproveBillingEvent() == null
+                        ? null
+                        : transferData.getServerApproveBillingEvent().getOfyKey())));
       } else {
         // There won't be a billing event, so we don't need a grace period
         builder.setGracePeriods(ImmutableSet.of());
@@ -506,24 +554,31 @@ public class DomainBase extends EppResource
   }
 
   /** Loads and returns the fully qualified host names of all linked nameservers. */
-  public ImmutableSortedSet<String> loadNameserverFullyQualifiedHostNames() {
+  public ImmutableSortedSet<String> loadNameserverHostNames() {
     return ofy()
         .load()
         .keys(getNameservers().stream().map(VKey::getOfyKey).collect(toImmutableSet()))
         .values()
         .stream()
-        .map(HostResource::getFullyQualifiedHostName)
+        .map(HostResource::getHostName)
         .collect(toImmutableSortedSet(Ordering.natural()));
   }
 
   /** A key to the registrant who registered this domain. */
-  public Key<ContactResource> getRegistrant() {
-    return nullToEmpty(allContacts)
-        .stream()
-        .filter(IS_REGISTRANT)
-        .findFirst()
-        .get()
-        .getContactKey();
+  public VKey<ContactResource> getRegistrant() {
+    return registrantContact;
+  }
+
+  public VKey<ContactResource> getAdminContact() {
+    return adminContact;
+  }
+
+  public VKey<ContactResource> getBillingContact() {
+    return billingContact;
+  }
+
+  public VKey<ContactResource> getTechContact() {
+    return techContact;
   }
 
   /** Associated contacts for the domain (other than registrant). */
@@ -539,7 +594,7 @@ public class DomainBase extends EppResource
   }
 
   /** Returns all referenced contacts from this domain or application. */
-  public ImmutableSet<Key<ContactResource>> getReferencedContacts() {
+  public ImmutableSet<VKey<ContactResource>> getReferencedContacts() {
     return nullToEmptyImmutableCopy(allContacts)
         .stream()
         .map(DesignatedContact::getContactKey)
@@ -551,6 +606,46 @@ public class DomainBase extends EppResource
     return tld;
   }
 
+  /**
+   * Sets the individual contact fields from {@code contacts}.
+   *
+   * <p>The registrant field is only set if {@code includeRegistrant} is true, as this field needs
+   * to be set in some circumstances but not in others.
+   */
+  private void setContactFields(Set<DesignatedContact> contacts, boolean includeRegistrant) {
+
+    // Set the individual contact fields.
+    for (DesignatedContact contact : contacts) {
+      switch (contact.getType()) {
+        case BILLING:
+          billingContact = contact.getContactKey();
+          break;
+        case TECH:
+          techContact = contact.getContactKey();
+          break;
+        case ADMIN:
+          adminContact = contact.getContactKey();
+          break;
+        case REGISTRANT:
+          if (includeRegistrant) {
+            registrantContact = contact.getContactKey();
+          }
+          break;
+        default:
+          throw new IllegalArgumentException("Unknown contact resource type: " + contact.getType());
+      }
+    }
+  }
+
+  @Override
+  public VKey<DomainBase> createVKey() {
+    return VKey.create(DomainBase.class, getRepoId(), Key.create(this));
+  }
+
+  public static VKey<DomainBase> createVKey(Key key) {
+    return VKey.create(DomainBase.class, key.getName(), key);
+  }
+
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
   private static final Predicate<DesignatedContact> IS_REGISTRANT =
       (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);
@@ -563,27 +658,19 @@ public class DomainBase extends EppResource
 
   /** A builder for constructing {@link DomainBase}, since it is immutable. */
   public static class Builder extends EppResource.Builder<DomainBase, Builder>
-      implements BuilderWithTransferData<Builder> {
+      implements BuilderWithTransferData<DomainTransferData, Builder> {
 
     public Builder() {}
 
     Builder(DomainBase instance) {
       super(instance);
-
-      // Convert nsHosts to nsHostVKeys.
-      if (instance.nsHosts != null) {
-        instance.nsHostVKeys =
-            instance.nsHosts.stream()
-                .map(key -> VKey.createOfy(HostResource.class, key))
-                .collect(toImmutableSet());
-      }
     }
 
     @Override
     public DomainBase build() {
       DomainBase instance = getInstance();
       // If TransferData is totally empty, set it to null.
-      if (TransferData.EMPTY.equals(getInstance().transferData)) {
+      if (DomainTransferData.EMPTY.equals(getInstance().transferData)) {
         setTransferData(null);
       }
       // A DomainBase has status INACTIVE if there are no nameservers.
@@ -594,17 +681,21 @@ public class DomainBase extends EppResource
       }
 
       checkArgumentNotNull(
-          emptyToNull(instance.fullyQualifiedDomainName), "Missing fullyQualifiedDomainName");
-      checkArgument(instance.allContacts.stream().anyMatch(IS_REGISTRANT), "Missing registrant");
+          emptyToNull(instance.fullyQualifiedDomainName), "Missing domainName");
+      if (instance.getRegistrant() == null
+          && instance.allContacts.stream().anyMatch(IS_REGISTRANT)) {
+        throw new IllegalArgumentException("registrant is null but is in allContacts");
+      }
+      checkArgumentNotNull(instance.getRegistrant(), "Missing registrant");
       instance.tld = getTldFromDomainName(instance.fullyQualifiedDomainName);
       return super.build();
     }
 
-    public Builder setFullyQualifiedDomainName(String fullyQualifiedDomainName) {
+    public Builder setDomainName(String domainName) {
       checkArgument(
-          fullyQualifiedDomainName.equals(canonicalizeDomainName(fullyQualifiedDomainName)),
+          domainName.equals(canonicalizeDomainName(domainName)),
           "Domain name must be in puny-coded, lower-case form");
-      getInstance().fullyQualifiedDomainName = fullyQualifiedDomainName;
+      getInstance().fullyQualifiedDomainName = domainName;
       return thisCastToDerived();
     }
 
@@ -613,11 +704,14 @@ public class DomainBase extends EppResource
       return thisCastToDerived();
     }
 
-    public Builder setRegistrant(Key<ContactResource> registrant) {
+    public Builder setRegistrant(VKey<ContactResource> registrant) {
       // Replace the registrant contact inside allContacts.
       getInstance().allContacts = union(
           getInstance().getContacts(),
           DesignatedContact.create(Type.REGISTRANT, checkArgumentNotNull(registrant)));
+
+      // Set the registrant field specifically.
+      getInstance().registrantContact = registrant;
       return thisCastToDerived();
     }
 
@@ -627,27 +721,12 @@ public class DomainBase extends EppResource
     }
 
     public Builder setNameservers(VKey<HostResource> nameserver) {
-      Optional<Key<HostResource>> nsKey = nameserver.maybeGetOfyKey();
-      if (nsKey.isPresent()) {
-        getInstance().nsHosts = ImmutableSet.of(nsKey.get());
-      } else {
-        getInstance().nsHosts = null;
-      }
-      getInstance().nsHostVKeys = ImmutableSet.of(nameserver);
+      getInstance().nsHosts = ImmutableSet.of(nameserver);
       return thisCastToDerived();
     }
 
     public Builder setNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
-      // If we have all of the ofy keys, we can set nsHosts.  Otherwise, make it null.
-      if (nameservers != null
-          && nameservers.stream().allMatch(key -> key.maybeGetOfyKey().isPresent())) {
-        getInstance().nsHosts =
-            nameservers.stream().map(key -> key.getOfyKey()).collect(toImmutableSet());
-      } else {
-        getInstance().nsHosts = null;
-      }
-
-      getInstance().nsHostVKeys = forceEmptyToNull(nameservers);
+      getInstance().nsHosts = forceEmptyToNull(nameservers);
       return thisCastToDerived();
     }
 
@@ -675,12 +754,16 @@ public class DomainBase extends EppResource
 
     public Builder setContacts(ImmutableSet<DesignatedContact> contacts) {
       checkArgument(contacts.stream().noneMatch(IS_REGISTRANT), "Registrant cannot be a contact");
+
       // Replace the non-registrant contacts inside allContacts.
       getInstance().allContacts =
           Streams.concat(
                   nullToEmpty(getInstance().allContacts).stream().filter(IS_REGISTRANT),
                   contacts.stream())
               .collect(toImmutableSet());
+
+      // Set the individual fields.
+      getInstance().setContactFields(contacts, false);
       return thisCastToDerived();
     }
 
@@ -761,7 +844,7 @@ public class DomainBase extends EppResource
     }
 
     @Override
-    public Builder setTransferData(TransferData transferData) {
+    public Builder setTransferData(DomainTransferData transferData) {
       getInstance().transferData = transferData;
       return thisCastToDerived();
     }

core/src/main/java/google/registry/model/domain/DomainCommand.java

@@ -30,7 +30,6 @@ import com.google.common.base.MoreObjects;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.contact.ContactResource;
@@ -41,6 +40,7 @@ import google.registry.model.eppinput.ResourceCommand.ResourceUpdate;
 import google.registry.model.eppinput.ResourceCommand.SingleResourceCommand;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
+import google.registry.persistence.VKey;
 import java.util.Map;
 import java.util.Set;
 import javax.annotation.Nullable;
@@ -71,18 +71,17 @@ public class DomainCommand {
     T cloneAndLinkReferences(DateTime now) throws InvalidReferencesException;
   }
 
-  /** The fields on "chgType" from {@link "http://tools.ietf.org/html/rfc5731"}. */
+  /** The fields on "chgType" from <a href="http://tools.ietf.org/html/rfc5731">RFC5731</a>. */
   @XmlTransient
-  public static class DomainCreateOrChange<B extends DomainBase.Builder>
-      extends ImmutableObject implements ResourceCreateOrChange<B> {
+  public static class DomainCreateOrChange<B extends DomainBase.Builder> extends ImmutableObject
+      implements ResourceCreateOrChange<B> {
 
     /** The contactId of the registrant who registered this domain. */
     @XmlElement(name = "registrant")
     String registrantContactId;
 
     /** A resolved key to the registrant who registered this domain. */
-    @XmlTransient
-    Key<ContactResource> registrant;
+    @XmlTransient VKey<ContactResource> registrant;
 
     /** Authorization info (aka transfer secret) of the domain. */
     DomainAuthInfo authInfo;
@@ -92,7 +91,7 @@ public class DomainCommand {
     }
 
     @Nullable
-    public Key<ContactResource> getRegistrant() {
+    public VKey<ContactResource> getRegistrant() {
       return registrant;
     }
 
@@ -102,19 +101,20 @@ public class DomainCommand {
   }
 
   /**
-   * A create command for a {@link DomainBase}, mapping "createType" from
-   * {@link "http://tools.ietf.org/html/rfc5731"}.
+   * A create command for a {@link DomainBase}, mapping "createType" from <a
+   * href="http://tools.ietf.org/html/rfc5731">RFC5731</a>.
    */
   @XmlRootElement
-  @XmlType(propOrder = {
-      "fullyQualifiedDomainName",
-      "period",
-      "nameserverFullyQualifiedHostNames",
-      "registrantContactId",
-      "foreignKeyedDesignatedContacts",
-      "authInfo"})
-  public static class Create
-      extends DomainCreateOrChange<DomainBase.Builder>
+  @XmlType(
+      propOrder = {
+        "fullyQualifiedDomainName",
+        "period",
+        "nameserverFullyQualifiedHostNames",
+        "registrantContactId",
+        "foreignKeyedDesignatedContacts",
+        "authInfo"
+      })
+  public static class Create extends DomainCreateOrChange<DomainBase.Builder>
       implements CreateOrUpdate<Create> {
 
     /** Fully qualified domain name, which serves as a unique identifier for this domain. */
@@ -127,8 +127,7 @@ public class DomainCommand {
     Set<String> nameserverFullyQualifiedHostNames;
 
     /** Resolved keys to hosts that are the nameservers for the domain. */
-    @XmlTransient
-    Set<Key<HostResource>> nameservers;
+    @XmlTransient Set<VKey<HostResource>> nameservers;
 
     /** Foreign keyed associated contacts for the domain (other than registrant). */
     @XmlElement(name = "contact")
@@ -158,7 +157,7 @@ public class DomainCommand {
       return nullToEmptyImmutableCopy(nameserverFullyQualifiedHostNames);
     }
 
-    public ImmutableSet<Key<HostResource>> getNameservers() {
+    public ImmutableSet<VKey<HostResource>> getNameservers() {
       return nullToEmptyImmutableCopy(nameservers);
     }
 
@@ -352,8 +351,7 @@ public class DomainCommand {
       Set<String> nameserverFullyQualifiedHostNames;
 
       /** Resolved keys to hosts that are the nameservers for the domain. */
-      @XmlTransient
-      Set<Key<HostResource>> nameservers;
+      @XmlTransient Set<VKey<HostResource>> nameservers;
 
       /** Foreign keyed associated contacts for the domain (other than registrant). */
       @XmlElement(name = "contact")
@@ -367,7 +365,7 @@ public class DomainCommand {
         return nullSafeImmutableCopy(nameserverFullyQualifiedHostNames);
       }
 
-      public ImmutableSet<Key<HostResource>> getNameservers() {
+      public ImmutableSet<VKey<HostResource>> getNameservers() {
         return nullToEmptyImmutableCopy(nameservers);
       }
 
@@ -417,7 +415,7 @@ public class DomainCommand {
     }
   }
 
-  private static Set<Key<HostResource>> linkHosts(
+  private static Set<VKey<HostResource>> linkHosts(
       Set<String> fullyQualifiedHostNames, DateTime now) throws InvalidReferencesException {
     if (fullyQualifiedHostNames == null) {
       return null;
@@ -435,18 +433,18 @@ public class DomainCommand {
     for (ForeignKeyedDesignatedContact contact : contacts) {
       foreignKeys.add(contact.contactId);
     }
-    ImmutableMap<String, Key<ContactResource>> loadedContacts =
+    ImmutableMap<String, VKey<ContactResource>> loadedContacts =
         loadByForeignKeysCached(foreignKeys.build(), ContactResource.class, now);
     ImmutableSet.Builder<DesignatedContact> linkedContacts = new ImmutableSet.Builder<>();
     for (ForeignKeyedDesignatedContact contact : contacts) {
-      linkedContacts.add(DesignatedContact.create(
-          contact.type, loadedContacts.get(contact.contactId)));
+      linkedContacts.add(
+          DesignatedContact.create(contact.type, loadedContacts.get(contact.contactId)));
     }
     return linkedContacts.build();
   }
 
   /** Loads keys to cached EPP resources by their foreign keys. */
-  private static <T extends EppResource> ImmutableMap<String, Key<T>> loadByForeignKeysCached(
+  private static <T extends EppResource> ImmutableMap<String, VKey<T>> loadByForeignKeysCached(
       final Set<String> foreignKeys, final Class<T> clazz, final DateTime now)
       throws InvalidReferencesException {
     Map<String, ForeignKeyIndex<T>> fkis = ForeignKeyIndex.loadCached(clazz, foreignKeys, now);

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -24,6 +24,7 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import org.joda.time.DateTime;
@@ -51,6 +52,7 @@ public class GracePeriod extends ImmutableObject {
   DateTime expirationTime;
 
   /** The registrar to bill. */
+  @Column(name = "registrarId")
   String clientId;
 
   /**

core/src/main/java/google/registry/model/domain/Period.java

@@ -16,15 +16,18 @@ package google.registry.model.domain;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlEnumValue;
 import javax.xml.bind.annotation.XmlValue;
 
-/** The "periodType" from {@link "http://tools.ietf.org/html/rfc5731"}. */
+/** The "periodType" from <a href="http://tools.ietf.org/html/rfc5731">RFC5731</a>. */
 @Embed
 @javax.persistence.Embeddable
 public class Period extends ImmutableObject {
 
+  @Enumerated(EnumType.STRING)
   @XmlAttribute
   Unit unit;
 

core/src/main/java/google/registry/model/domain/fee/BaseFee.java

@@ -104,6 +104,8 @@ public abstract class BaseFee extends ImmutableObject {
 
   @XmlTransient Range<DateTime> validDateRange;
 
+  @XmlTransient boolean isPremium;
+
   public String getDescription() {
     return description;
   }
@@ -120,6 +122,11 @@ public abstract class BaseFee extends ImmutableObject {
     return firstNonNull(refundable, true);
   }
 
+  /** Returns whether the fee in question is a premium price. */
+  public boolean isPremium() {
+    return isPremium;
+  }
+
   /**
    * According to the fee extension specification, a fee must always be non-negative, while a credit
    * must always be negative. Essentially, they are the same thing, just with different sign.

core/src/main/java/google/registry/model/domain/fee/Fee.java

@@ -31,25 +31,33 @@ import org.joda.time.DateTime;
 public class Fee extends BaseFee {
 
   /** Creates a Fee for the given cost and type with the default description. */
-  public static Fee create(BigDecimal cost, FeeType type, Object... descriptionArgs) {
+  public static Fee create(
+      BigDecimal cost, FeeType type, boolean isPremium, Object... descriptionArgs) {
     checkArgumentNotNull(type, "Must specify the type of the fee");
-    return createWithCustomDescription(cost, type, type.renderDescription(descriptionArgs));
+    return createWithCustomDescription(
+        cost, type, isPremium, type.renderDescription(descriptionArgs));
   }
 
   /** Creates a Fee for the given cost, type, and valid date range with the default description. */
   public static Fee create(
-      BigDecimal cost, FeeType type, Range<DateTime> validDateRange, Object... descriptionArgs) {
-    Fee instance = create(cost, type, descriptionArgs);
+      BigDecimal cost,
+      FeeType type,
+      boolean isPremium,
+      Range<DateTime> validDateRange,
+      Object... descriptionArgs) {
+    Fee instance = create(cost, type, isPremium, descriptionArgs);
     instance.validDateRange = validDateRange;
     return instance;
   }
 
   /** Creates a Fee for the given cost and type with a custom description. */
-  public static Fee createWithCustomDescription(BigDecimal cost, FeeType type, String description) {
+  private static Fee createWithCustomDescription(
+      BigDecimal cost, FeeType type, boolean isPremium, String description) {
     Fee instance = new Fee();
     instance.cost = checkNotNull(cost);
-    checkArgument(instance.cost.signum() >= 0);
+    checkArgument(instance.cost.signum() >= 0, "Cost must be a positive number");
     instance.type = checkNotNull(type);
+    instance.isPremium = isPremium;
     instance.description = description;
     return instance;
   }

core/src/main/java/google/registry/model/domain/fee11/FeeCheckCommandExtensionV11.java

@@ -56,7 +56,7 @@ public class FeeCheckCommandExtensionV11 extends ImmutableObject
   /** The period to check. */
   Period period;
 
-  /** The class to check. */
+  /** The fee class to check. */
   @XmlElement(name = "class")
   String feeClass;
 

core/src/main/java/google/registry/model/domain/fee12/FeeCheckCommandExtensionItemV12.java

@@ -29,11 +29,13 @@ import org.joda.time.DateTime;
  * An individual price check item in version 0.12 of the fee extension on domain check commands.
  * Items look like:
  *
+ * <pre>{@code
  * <fee:command name="renew" phase="sunrise" subphase="hello">
  *   <fee:period unit="y">1</fee:period>
  *   <fee:class>premium</fee:class>
  *   <fee:date>2017-05-17T13:22:21.0Z</fee:date>
  * </fee:command>
+ * }</pre>
  *
  * In a change from previous versions of the extension, items do not contain domain names; instead,
  * the names from the non-extension check element are used.

core/src/main/java/google/registry/model/domain/launch/LaunchNotice.java

@@ -95,9 +95,6 @@ public class LaunchNotice extends ImmutableObject {
 
   /**
    * Validate the checksum of the notice against the domain label.
-   *
-   * @throws IllegalArgumentException
-   * @throws InvalidChecksumException
    */
   public void validate(String domainLabel) throws InvalidChecksumException {
     // According to http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3, a TCNID

core/src/main/java/google/registry/model/domain/rgp/GracePeriodStatus.java

@@ -49,9 +49,9 @@ public enum GracePeriodStatus implements EppEnum {
   AUTO_RENEW("autoRenewPeriod"),
 
   /**
-   * This status value is used to describe a domain for which a <delete> command has been received,
-   * but the domain has not yet been purged because an opportunity exists to restore the domain and
-   * abort the deletion process.
+   * This status value is used to describe a domain for which a &lt;delete&gt; command has been
+   * received, but the domain has not yet been purged because an opportunity exists to restore the
+   * domain and abort the deletion process.
    */
   REDEMPTION("redemptionPeriod"),
 

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -45,6 +45,8 @@ import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.common.TimedTransitionProperty.TimeMapper;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
@@ -53,6 +55,7 @@ import org.joda.time.DateTime;
 /** An entity representing an allocation token. */
 @ReportedOn
 @Entity
+@WithStringVKey
 public class AllocationToken extends BackupGroupRoot implements Buildable {
 
   // Promotions should only move forward, and ENDED / CANCELLED are terminal states.
@@ -179,6 +182,10 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
     return tokenStatusTransitions;
   }
 
+  public VKey<AllocationToken> createVKey() {
+    return VKey.createOfy(getClass(), Key.create(this));
+  }
+
   @Override
   public Builder asBuilder() {
     return new Builder(clone(this));

core/src/main/java/google/registry/model/domain/token/AllocationTokenExtension.java

@@ -25,7 +25,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
  * An allocation token extension that may be present on EPP domain commands.
  *
  * @see <a href="https://tools.ietf.org/html/draft-ietf-regext-allocation-token-04">the IETF
- *     draft</a> for full details.
+ *     draft</a>
  */
 @XmlRootElement(name = "allocationToken")
 public class AllocationTokenExtension extends ImmutableObject implements CommandExtension {

core/src/main/java/google/registry/model/eppcommon/Address.java

@@ -44,8 +44,9 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 /**
  * Container for generic street address.
  *
- * <p>This is the "addrType" type from {@link "http://tools.ietf.org/html/rfc5733"}. It also matches
- * the "addrType" type from {@link "http://tools.ietf.org/html/draft-lozano-tmch-smd"}.
+ * <p>This is the "addrType" type from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. It
+ * also matches the "addrType" type from <a
+ * href="http://tools.ietf.org/html/draft-lozano-tmch-smd">Mark and Signed Mark Objects Mapping</a>.
  *
  * @see google.registry.model.contact.ContactAddress
  * @see google.registry.model.mark.MarkAddress

core/src/main/java/google/registry/model/eppcommon/PhoneNumber.java

@@ -29,8 +29,9 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 /**
  * Container for generic E164 phone number.
  *
- * <p>This is the "e164" type from {@link "http://tools.ietf.org/html/rfc5733"}. It also matches the
- * "e164Type" type from {@link "http://tools.ietf.org/html/draft-lozano-tmch-smd"}.
+ * <p>This is the "e164" type from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. It also
+ * matches the "e164Type" type from <a href="http://tools.ietf.org/html/draft-lozano-tmch-smd">Mark
+ * and Signed Mark Objects Mapping</a>
  *
  * <blockquote>
  *

core/src/main/java/google/registry/model/eppcommon/StatusValue.java

@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableSet;
 import google.registry.model.EppResource;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.host.HostBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.translators.EnumToAttributeAdapter.EppEnum;
 import google.registry.model.translators.StatusValueAdapter;
@@ -127,7 +128,7 @@ public enum StatusValue implements EppEnum {
 
   /** Enum to help clearly list which resource types a status value is allowed to be present on. */
   private enum AllowedOn {
-    ALL(ContactResource.class, DomainBase.class, HostResource.class),
+    ALL(ContactResource.class, DomainBase.class, HostBase.class,  HostResource.class),
     NONE,
     DOMAINS(DomainBase.class);
 

core/src/main/java/google/registry/model/eppoutput/Greeting.java

@@ -24,7 +24,7 @@ import javax.xml.bind.annotation.XmlElementWrapper;
 import org.joda.time.DateTime;
 
 /**
- * A greeting, defined in {@link "http://tools.ietf.org/html/rfc5730"}.
+ * A greeting, defined in <a href="http://tools.ietf.org/html/rfc5730">RFC5730</a>.
  *
  * <p>It would be nice to make this a singleton, but we need the {@link #svDate} field to stay
  * current.

core/src/main/java/google/registry/model/host/HostBase.java

@@ -0,0 +1,233 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.host;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Sets.difference;
+import static com.google.common.collect.Sets.union;
+import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.IgnoreSave;
+import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.condition.IfNull;
+import google.registry.model.EppResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.transfer.TransferData;
+import google.registry.persistence.VKey;
+import java.net.InetAddress;
+import java.util.Optional;
+import java.util.Set;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.MappedSuperclass;
+import org.joda.time.DateTime;
+
+/**
+ * A persistable Host resource including mutable and non-mutable fields.
+ *
+ * <p>A host's {@link TransferData} is stored on the superordinate domain. Non-subordinate hosts
+ * don't carry a full set of TransferData; all they have is lastTransferTime.
+ *
+ * <p>This class deliberately does not include an {@link javax.persistence.Id} so that any
+ * foreign-keyed fields can refer to the proper parent entity's ID, whether we're storing this in
+ * the DB itself or as part of another entity
+ *
+ * @see <a href="https://tools.ietf.org/html/rfc5732">RFC 5732</a>
+ */
+@MappedSuperclass
+@Embeddable
+@Access(AccessType.FIELD)
+public class HostBase extends EppResource {
+
+  /**
+   * Fully qualified hostname, which is a unique identifier for this host.
+   *
+   * <p>This is only unique in the sense that for any given lifetime specified as the time range
+   * from (creationTime, deletionTime) there can only be one host in Datastore with this name.
+   * However, there can be many hosts with the same name and non-overlapping lifetimes.
+   */
+  // TODO(b/158858642): Rename this to hostName when we are off Datastore
+  @Index
+  @Column(name = "hostName")
+  String fullyQualifiedHostName;
+
+  /** IP Addresses for this host. Can be null if this is an external host. */
+  @Index Set<InetAddress> inetAddresses;
+
+  /** The superordinate domain of this host, or null if this is an external host. */
+  @Index
+  @IgnoreSave(IfNull.class)
+  @DoNotHydrate
+  VKey<DomainBase> superordinateDomain;
+
+  /**
+   * The time that this resource was last transferred.
+   *
+   * <p>Can be null if the resource has never been transferred.
+   */
+  DateTime lastTransferTime;
+
+  /**
+   * The most recent time that the {@link #superordinateDomain} field was changed.
+   *
+   * <p>This should be updated whenever the superordinate domain changes, including when it is set
+   * to null. This field will be null for new hosts that have never experienced a change of
+   * superordinate domain.
+   */
+  DateTime lastSuperordinateChange;
+
+  public String getHostName() {
+    return fullyQualifiedHostName;
+  }
+
+  public VKey<DomainBase> getSuperordinateDomain() {
+    return superordinateDomain;
+  }
+
+  public boolean isSubordinate() {
+    return superordinateDomain != null;
+  }
+
+  public ImmutableSet<InetAddress> getInetAddresses() {
+    return nullToEmptyImmutableCopy(inetAddresses);
+  }
+
+  public DateTime getLastTransferTime() {
+    return lastTransferTime;
+  }
+
+  public DateTime getLastSuperordinateChange() {
+    return lastSuperordinateChange;
+  }
+
+  @Override
+  public String getForeignKey() {
+    return fullyQualifiedHostName;
+  }
+
+  @Override
+  public VKey<? extends EppResource> createVKey() {
+    return VKey.createOfy(HostBase.class, Key.create(this));
+  }
+
+  @Deprecated
+  @Override
+  public HostBase cloneProjectedAtTime(DateTime now) {
+    return this;
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder<>(clone(this));
+  }
+
+  /**
+   * Compute the correct last transfer time for this host given its loaded superordinate domain.
+   *
+   * <p>Hosts can move between superordinate domains, so to know which lastTransferTime is correct
+   * we need to know if the host was attached to this superordinate the last time that the
+   * superordinate was transferred. If the last superordinate change was before this time, then the
+   * host was attached to this superordinate domain during that transfer.
+   *
+   * <p>If the host is not subordinate the domain can be null and we just return last transfer time.
+   *
+   * @param superordinateDomain the loaded superordinate domain, which must match the key in the
+   *     {@link #superordinateDomain} field. Passing it as a parameter allows the caller to control
+   *     the degree of consistency used to load it.
+   */
+  public DateTime computeLastTransferTime(@Nullable DomainBase superordinateDomain) {
+    if (!isSubordinate()) {
+      checkArgument(superordinateDomain == null);
+      return getLastTransferTime();
+    }
+    checkArgument(
+        superordinateDomain != null
+            && superordinateDomain.createVKey().equals(getSuperordinateDomain()));
+    DateTime lastSuperordinateChange =
+        Optional.ofNullable(getLastSuperordinateChange()).orElse(getCreationTime());
+    DateTime lastTransferOfCurrentSuperordinate =
+        Optional.ofNullable(superordinateDomain.getLastTransferTime()).orElse(START_OF_TIME);
+    return lastSuperordinateChange.isBefore(lastTransferOfCurrentSuperordinate)
+        ? superordinateDomain.getLastTransferTime()
+        : getLastTransferTime();
+  }
+
+  /** A builder for constructing {@link HostBase}, since it is immutable. */
+  protected static class Builder<T extends HostBase, B extends Builder<T, B>>
+      extends EppResource.Builder<T, B> {
+    public Builder() {}
+
+    protected Builder(T instance) {
+      super(instance);
+    }
+
+    // Strangely, if we don't add these @Overrides the methods return an EppResource.Builder
+    // even though we parameterize it with B in both cases anyway.
+    @Override
+    public B setRepoId(String repoId) {
+      return super.setRepoId(repoId);
+    }
+
+    @Override
+    public T build() {
+      return super.build();
+    }
+
+    public B setHostName(String hostName) {
+      checkArgument(
+          hostName.equals(canonicalizeDomainName(hostName)),
+          "Host name must be in puny-coded, lower-case form");
+      getInstance().fullyQualifiedHostName = hostName;
+      return thisCastToDerived();
+    }
+
+    public B setInetAddresses(ImmutableSet<InetAddress> inetAddresses) {
+      getInstance().inetAddresses = inetAddresses;
+      return thisCastToDerived();
+    }
+
+    public B setLastSuperordinateChange(DateTime lastSuperordinateChange) {
+      getInstance().lastSuperordinateChange = lastSuperordinateChange;
+      return thisCastToDerived();
+    }
+
+    public B addInetAddresses(ImmutableSet<InetAddress> inetAddresses) {
+      return setInetAddresses(
+          ImmutableSet.copyOf(union(getInstance().getInetAddresses(), inetAddresses)));
+    }
+
+    public B removeInetAddresses(ImmutableSet<InetAddress> inetAddresses) {
+      return setInetAddresses(
+          ImmutableSet.copyOf(difference(getInstance().getInetAddresses(), inetAddresses)));
+    }
+
+    public B setSuperordinateDomain(VKey<DomainBase> superordinateDomain) {
+      getInstance().superordinateDomain = superordinateDomain;
+      return thisCastToDerived();
+    }
+
+    public B setLastTransferTime(DateTime lastTransferTime) {
+      getInstance().lastTransferTime = lastTransferTime;
+      return thisCastToDerived();
+    }
+  }
+}

core/src/main/java/google/registry/model/host/HostCommand.java

@@ -32,7 +32,7 @@ import javax.xml.bind.annotation.XmlType;
 /** A collection of {@link HostResource} commands. */
 public class HostCommand {
 
-  /** The fields on "chgType" from {@link "http://tools.ietf.org/html/rfc5732"}. */
+  /** The fields on "chgType" from <a href="http://tools.ietf.org/html/rfc5732">RFC5732</a>. */
   @XmlTransient
   abstract static class HostCreateOrChange extends AbstractSingleResourceCommand
       implements ResourceCreateOrChange<HostResource.Builder> {
@@ -42,13 +42,13 @@ public class HostCommand {
   }
 
   /**
-   * A create command for a {@link HostResource}, mapping "createType" from
-   * {@link "http://tools.ietf.org/html/rfc5732"}.
+   * A create command for a {@link HostResource}, mapping "createType" from <a
+   * href="http://tools.ietf.org/html/rfc5732">RFC5732</a>.
    */
-  @XmlType(propOrder = {"targetId", "inetAddresses" })
+  @XmlType(propOrder = {"targetId", "inetAddresses"})
   @XmlRootElement
-  public static class Create
-      extends HostCreateOrChange implements ResourceCreateOrChange<HostResource.Builder> {
+  public static class Create extends HostCreateOrChange
+      implements ResourceCreateOrChange<HostResource.Builder> {
     /** IP Addresses for this host. Can be null if this is an external host. */
     @XmlElement(name = "addr")
     Set<InetAddress> inetAddresses;

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -0,0 +1,90 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.host;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.EppResource;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+
+/**
+ * A persisted history entry representing an EPP modification to a host.
+ *
+ * <p>In addition to the general history fields (e.g. action time, registrar ID) we also persist a
+ * copy of the host entity at this point in time. We persist a raw {@link HostBase} so that the
+ * foreign-keyed fields in that class can refer to this object.
+ */
+@Entity
+@javax.persistence.Table(
+    indexes = {
+      @javax.persistence.Index(columnList = "creationTime"),
+      @javax.persistence.Index(columnList = "historyRegistrarId"),
+      @javax.persistence.Index(columnList = "hostName"),
+      @javax.persistence.Index(columnList = "historyType"),
+      @javax.persistence.Index(columnList = "historyModificationTime")
+    })
+public class HostHistory extends HistoryEntry {
+
+  // Store HostBase instead of HostResource so we don't pick up its @Id
+  HostBase hostBase;
+
+  @Column(nullable = false)
+  VKey<HostResource> hostRepoId;
+
+  /** The state of the {@link HostBase} object at this point in time. */
+  public HostBase getHostBase() {
+    return hostBase;
+  }
+
+  /** The key to the {@link google.registry.model.host.HostResource} this is based off of. */
+  public VKey<HostResource> getHostRepoId() {
+    return hostRepoId;
+  }
+
+  @Override
+  public Builder asBuilder() {
+    return new Builder(clone(this));
+  }
+
+  public static class Builder extends HistoryEntry.Builder<HostHistory, Builder> {
+
+    public Builder() {}
+
+    public Builder(HostHistory instance) {
+      super(instance);
+    }
+
+    public Builder setHostBase(HostBase hostBase) {
+      getInstance().hostBase = hostBase;
+      return this;
+    }
+
+    public Builder setHostResourceId(VKey<HostResource> hostRepoId) {
+      getInstance().hostRepoId = hostRepoId;
+      hostRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
+      return this;
+    }
+
+    // We can remove this once all HistoryEntries are converted to History objects
+    @Override
+    public Builder setParent(Key<? extends EppResource> parent) {
+      super.setParent(parent);
+      getInstance().hostRepoId = VKey.createOfy(HostResource.class, (Key<HostResource>) parent);
+      return this;
+    }
+  }
+}

core/src/main/java/google/registry/model/host/HostResource.java

@@ -14,209 +14,55 @@
 
 package google.registry.model.host;
 
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.Sets.difference;
-import static com.google.common.collect.Sets.union;
-import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
-import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
-
-import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
-import com.googlecode.objectify.annotation.IgnoreSave;
-import com.googlecode.objectify.annotation.Index;
-import com.googlecode.objectify.condition.IfNull;
-import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.transfer.TransferData;
 import google.registry.persistence.VKey;
-import google.registry.persistence.converter.VKeyConverter;
-import java.net.InetAddress;
-import java.util.Optional;
-import java.util.Set;
-import javax.annotation.Nullable;
-import javax.persistence.ElementCollection;
-import org.joda.time.DateTime;
+import google.registry.persistence.WithStringVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 
 /**
  * A persistable Host resource including mutable and non-mutable fields.
  *
- * <p>A host's {@link TransferData} is stored on the superordinate domain. Non-subordinate hosts
- * don't carry a full set of TransferData; all they have is lastTransferTime.
- *
- * @see <a href="https://tools.ietf.org/html/rfc5732">RFC 5732</a>
+ * <p>The {@link javax.persistence.Id} of the HostResource is the repoId.
  */
 @ReportedOn
 @Entity
 @javax.persistence.Entity
 @ExternalMessagingName("host")
-public class HostResource extends EppResource implements ForeignKeyedEppResource {
+@WithStringVKey
+@Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property)
+public class HostResource extends HostBase
+    implements DatastoreAndSqlEntity, ForeignKeyedEppResource {
 
-  /**
-   * Fully qualified hostname, which is a unique identifier for this host.
-   *
-   * <p>This is only unique in the sense that for any given lifetime specified as the time range
-   * from (creationTime, deletionTime) there can only be one host in Datastore with this name.
-   * However, there can be many hosts with the same name and non-overlapping lifetimes.
-   */
-  @Index
-  String fullyQualifiedHostName;
-
-  /** IP Addresses for this host. Can be null if this is an external host. */
-  @Index @ElementCollection Set<InetAddress> inetAddresses;
-
-  /** The superordinate domain of this host, or null if this is an external host. */
-  @Index
-  @IgnoreSave(IfNull.class)
-  @DoNotHydrate
-  Key<DomainBase> superordinateDomain;
-
-  /**
-   * The time that this resource was last transferred.
-   *
-   * <p>Can be null if the resource has never been transferred.
-   */
-  DateTime lastTransferTime;
-
-  /**
-   * The most recent time that the {@link #superordinateDomain} field was changed.
-   *
-   * <p>This should be updated whenever the superordinate domain changes, including when it is set
-   * to null. This field will be null for new hosts that have never experienced a change of
-   * superordinate domain.
-   */
-  DateTime lastSuperordinateChange;
-
-  public String getFullyQualifiedHostName() {
-    return fullyQualifiedHostName;
-  }
-
-  public Key<DomainBase> getSuperordinateDomain() {
-    return superordinateDomain;
-  }
-
-  public boolean isSubordinate() {
-    return superordinateDomain != null;
-  }
-
-  public ImmutableSet<InetAddress> getInetAddresses() {
-    return nullToEmptyImmutableCopy(inetAddresses);
-  }
-
-  public DateTime getLastTransferTime() {
-    return lastTransferTime;
-  }
-
-  public DateTime getLastSuperordinateChange() {
-    return lastSuperordinateChange;
+  @Override
+  @javax.persistence.Id
+  @Access(AccessType.PROPERTY) // to tell it to use the non-default property-as-ID
+  public String getRepoId() {
+    return super.getRepoId();
   }
 
   @Override
-  public String getForeignKey() {
-    return fullyQualifiedHostName;
-  }
-
-  public VKey<HostResource> createKey() {
+  public VKey<HostResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.createOfy(HostResource.class, Key.create(this));
   }
 
-  @Deprecated
-  @Override
-  public HostResource cloneProjectedAtTime(DateTime now) {
-    return this;
-  }
-
-  /**
-   * Compute the correct last transfer time for this host given its loaded superordinate domain.
-   *
-   * <p>Hosts can move between superordinate domains, so to know which lastTransferTime is correct
-   * we need to know if the host was attached to this superordinate the last time that the
-   * superordinate was transferred. If the last superordinate change was before this time, then the
-   * host was attached to this superordinate domain during that transfer.
-   *
-   * <p>If the host is not subordinate the domain can be null and we just return last transfer time.
-   *
-   * @param superordinateDomain the loaded superordinate domain, which must match the key in
-   *     the {@link #superordinateDomain} field. Passing it as a parameter allows the caller to
-   *     control the degree of consistency used to load it.
-   */
-   public DateTime computeLastTransferTime(@Nullable DomainBase superordinateDomain) {
-    if (!isSubordinate()) {
-      checkArgument(superordinateDomain == null);
-      return getLastTransferTime();
-    }
-    checkArgument(
-        superordinateDomain != null
-            && Key.create(superordinateDomain).equals(getSuperordinateDomain()));
-    DateTime lastSuperordinateChange =
-        Optional.ofNullable(getLastSuperordinateChange()).orElse(getCreationTime());
-    DateTime lastTransferOfCurrentSuperordinate =
-        Optional.ofNullable(superordinateDomain.getLastTransferTime()).orElse(START_OF_TIME);
-    return lastSuperordinateChange.isBefore(lastTransferOfCurrentSuperordinate)
-        ? superordinateDomain.getLastTransferTime()
-        : getLastTransferTime();
-  }
-
   @Override
   public Builder asBuilder() {
     return new Builder(clone(this));
   }
 
   /** A builder for constructing {@link HostResource}, since it is immutable. */
-  public static class Builder extends EppResource.Builder<HostResource, Builder> {
+  public static class Builder extends HostBase.Builder<HostResource, Builder> {
     public Builder() {}
 
     private Builder(HostResource instance) {
       super(instance);
     }
-
-    public Builder setFullyQualifiedHostName(String fullyQualifiedHostName) {
-      checkArgument(
-          fullyQualifiedHostName.equals(canonicalizeDomainName(fullyQualifiedHostName)),
-          "Host name must be in puny-coded, lower-case form");
-      getInstance().fullyQualifiedHostName = fullyQualifiedHostName;
-      return this;
-    }
-
-    public Builder setInetAddresses(ImmutableSet<InetAddress> inetAddresses) {
-      getInstance().inetAddresses = inetAddresses;
-      return this;
-    }
-
-    public Builder setLastSuperordinateChange(DateTime lastSuperordinateChange) {
-      getInstance().lastSuperordinateChange = lastSuperordinateChange;
-      return this;
-    }
-
-    public Builder addInetAddresses(ImmutableSet<InetAddress> inetAddresses) {
-      return setInetAddresses(ImmutableSet.copyOf(
-          union(getInstance().getInetAddresses(), inetAddresses)));
-    }
-
-    public Builder removeInetAddresses(ImmutableSet<InetAddress> inetAddresses) {
-      return setInetAddresses(ImmutableSet.copyOf(
-          difference(getInstance().getInetAddresses(), inetAddresses)));
-    }
-
-    public Builder setSuperordinateDomain(Key<DomainBase> superordinateDomain) {
-      getInstance().superordinateDomain = superordinateDomain;
-      return this;
-    }
-
-    public Builder setLastTransferTime(DateTime lastTransferTime) {
-      getInstance().lastTransferTime = lastTransferTime;
-      return this;
-    }
-  }
-
-  public static class VKeyHostResourceConverter extends VKeyConverter<HostResource> {
-    @Override
-    protected Class<HostResource> getAttributeClass() {
-      return HostResource.class;
-    }
   }
 }

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -43,6 +43,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
 import java.util.Optional;
@@ -99,7 +100,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * <p>This field holds a key to the only referenced resource. It is named "topReference" for
    * historical reasons.
    */
-  Key<E> topReference;
+  VKey<E> topReference;
 
   public String getForeignKey() {
     return foreignKey;
@@ -109,7 +110,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     return deletionTime;
   }
 
-  public Key<E> getResourceKey() {
+  public VKey<E> getResourceKey() {
     return topReference;
   }
 
@@ -125,7 +126,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     @SuppressWarnings("unchecked")
     Class<E> resourceClass = (Class<E>) resource.getClass();
     ForeignKeyIndex<E> instance = instantiate(mapToFkiClass(resourceClass));
-    instance.topReference = Key.create(resource);
+    instance.topReference = (VKey<E>) resource.createVKey();
     instance.foreignKey = resource.getForeignKey();
     instance.deletionTime = deletionTime;
     return instance;
@@ -141,18 +142,18 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   /**
    * Loads a {@link Key} to an {@link EppResource} from Datastore by foreign key.
    *
-   * <p>Returns null if no foreign key index with this foreign key was ever created, or if the
-   * most recently created foreign key index was deleted before time "now". This method does not
-   * actually check that the referenced resource actually exists. However, for normal epp resources,
-   * it is safe to assume that the referenced resource exists if the foreign key index does.
+   * <p>Returns null if no foreign key index with this foreign key was ever created, or if the most
+   * recently created foreign key index was deleted before time "now". This method does not actually
+   * check that the referenced resource actually exists. However, for normal epp resources, it is
+   * safe to assume that the referenced resource exists if the foreign key index does.
    *
    * @param clazz the resource type to load
    * @param foreignKey id to match
    * @param now the current logical time to use when checking for soft deletion of the foreign key
-   *        index
+   *     index
    */
   @Nullable
-  public static <E extends EppResource> Key<E> loadAndGetKey(
+  public static <E extends EppResource> VKey<E> loadAndGetKey(
       Class<E> clazz, String foreignKey, DateTime now) {
     ForeignKeyIndex<E> index = load(clazz, foreignKey, now);
     return (index == null) ? null : index.getResourceKey();

core/src/main/java/google/registry/model/ofy/CommitLogBucket.java

@@ -22,6 +22,7 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.collect.ContiguousSet;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Range;
@@ -33,6 +34,8 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.util.NonFinalForTesting;
 import java.util.Random;
 import java.util.function.Supplier;
@@ -41,17 +44,16 @@ import org.joda.time.DateTime;
 /**
  * Root for a random commit log bucket.
  *
- * <p>This is used to shard {@link CommitLogManifest} objects into
- * {@link RegistryConfig#getCommitLogBucketCount() N} entity groups. This increases
- * transaction throughput, while maintaining the ability to perform strongly-consistent ancestor
- * queries.
+ * <p>This is used to shard {@link CommitLogManifest} objects into {@link
+ * RegistryConfig#getCommitLogBucketCount() N} entity groups. This increases transaction throughput,
+ * while maintaining the ability to perform strongly-consistent ancestor queries.
  *
  * @see <a href="https://cloud.google.com/appengine/articles/scaling/contention">Avoiding Datastore
  *     contention</a>
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogBucket extends ImmutableObject implements Buildable {
+public class CommitLogBucket extends ImmutableObject implements Buildable, DatastoreEntity {
 
   /**
    * Ranges from 1 to {@link RegistryConfig#getCommitLogBucketCount()}, inclusive; starts at 1 since
@@ -70,6 +72,11 @@ public class CommitLogBucket extends ImmutableObject implements Buildable {
     return lastWrittenTime;
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   /**
    * Returns the key for the specified bucket ID.
    *

core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java

@@ -27,6 +27,8 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -36,15 +38,15 @@ import org.joda.time.DateTime;
  * Entity representing a point-in-time consistent view of Datastore, based on commit logs.
  *
  * <p>Conceptually, this entity consists of two pieces of information: the checkpoint "wall" time
- * and a set of bucket checkpoint times.  The former is the ID for this checkpoint (constrained
- * to be unique upon checkpoint creation) and also represents the approximate wall time of the
- * consistent Datastore view this checkpoint represents.  The latter is really a mapping from
- * bucket ID to timestamp, where the timestamp dictates the upper bound (inclusive) on commit logs
- * from that bucket to include when restoring Datastore to this checkpoint.
+ * and a set of bucket checkpoint times. The former is the ID for this checkpoint (constrained to be
+ * unique upon checkpoint creation) and also represents the approximate wall time of the consistent
+ * Datastore view this checkpoint represents. The latter is really a mapping from bucket ID to
+ * timestamp, where the timestamp dictates the upper bound (inclusive) on commit logs from that
+ * bucket to include when restoring Datastore to this checkpoint.
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogCheckpoint extends ImmutableObject {
+public class CommitLogCheckpoint extends ImmutableObject implements DatastoreEntity {
 
   /** Shared singleton parent entity for commit log checkpoints. */
   @Parent
@@ -71,6 +73,11 @@ public class CommitLogCheckpoint extends ImmutableObject {
     return builder.build();
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   /**
    * Creates a CommitLogCheckpoint for the given wall time and bucket checkpoint times, specified as
    * a map from bucket ID to bucket commit timestamp.

core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java

@@ -17,20 +17,21 @@ package google.registry.model.ofy;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import org.joda.time.DateTime;
 
-/**
- * Singleton parent entity for all commit log checkpoints.
- */
+/** Singleton parent entity for all commit log checkpoints. */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogCheckpointRoot extends ImmutableObject {
+public class CommitLogCheckpointRoot extends ImmutableObject implements DatastoreEntity {
 
   public static final long SINGLETON_ID = 1;  // There is always exactly one of these.
 
@@ -49,6 +50,11 @@ public class CommitLogCheckpointRoot extends ImmutableObject {
     return lastWrittenTime;
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   public static CommitLogCheckpointRoot loadRoot() {
     CommitLogCheckpointRoot root = ofy().load().key(getKey()).now();
     return root == null ? new CommitLogCheckpointRoot() : root;

core/src/main/java/google/registry/model/ofy/CommitLogManifest.java

@@ -17,6 +17,7 @@ package google.registry.model.ofy;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static org.joda.time.DateTimeZone.UTC;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -25,6 +26,8 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.LinkedHashSet;
 import java.util.Set;
 import org.joda.time.DateTime;
@@ -38,7 +41,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogManifest extends ImmutableObject {
+public class CommitLogManifest extends ImmutableObject implements DatastoreEntity {
 
   /** Commit log manifests are parented on a random bucket. */
   @Parent
@@ -67,6 +70,11 @@ public class CommitLogManifest extends ImmutableObject {
     return nullToEmptyImmutableCopy(deletions);
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   public static CommitLogManifest create(
       Key<CommitLogBucket> parent, DateTime commitTime, Set<Key<?>> deletions) {
     CommitLogManifest instance = new CommitLogManifest();

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -21,6 +21,7 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 
 import com.google.appengine.api.datastore.KeyFactory;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -28,11 +29,13 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /** Representation of a saved entity in a {@link CommitLogManifest} (not deletes). */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogMutation extends ImmutableObject {
+public class CommitLogMutation extends ImmutableObject implements DatastoreEntity {
 
   /** The manifest this belongs to. */
   @Parent
@@ -58,6 +61,11 @@ public class CommitLogMutation extends ImmutableObject {
     return createFromPbBytes(entityProtoBytes);
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   /**
    * Returns a new mutation entity created from an @Entity ImmutableObject instance.
    *

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -14,7 +14,9 @@
 
 package google.registry.model.ofy;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
@@ -22,6 +24,7 @@ import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.TransactionManager;
 import java.util.Iterator;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.StreamSupport;
@@ -93,42 +96,45 @@ public class DatastoreTransactionManager implements TransactionManager {
 
   @Override
   public void saveNew(Object entity) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    checkArgumentNotNull(entity, "entity must be specified");
+    getOfy().save().entity(entity);
   }
 
   @Override
   public void saveAllNew(ImmutableCollection<?> entities) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    getOfy().save().entities(entities);
   }
 
   @Override
   public void saveNewOrUpdate(Object entity) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    checkArgumentNotNull(entity, "entity must be specified");
+    getOfy().save().entity(entity);
   }
 
   @Override
   public void saveNewOrUpdateAll(ImmutableCollection<?> entities) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    getOfy().save().entities(entities);
   }
 
   @Override
   public void update(Object entity) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    checkArgumentNotNull(entity, "entity must be specified");
+    getOfy().save().entity(entity);
   }
 
   @Override
   public void updateAll(ImmutableCollection<?> entities) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    getOfy().save().entities(entities);
   }
 
   @Override
   public boolean checkExists(Object entity) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    return getOfy().load().key(Key.create(entity)).now() != null;
   }
 
   @Override
   public <T> boolean checkExists(VKey<T> key) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    return getOfy().load().key(key.getOfyKey()).now() != null;
   }
 
   // TODO: add tests for these methods.  They currently have some degree of test coverage because
@@ -136,14 +142,23 @@ public class DatastoreTransactionManager implements TransactionManager {
   // VKey instead of by ofy Key.  But ideally, there should be one set of TransactionManager
   // interface tests that are applied to both the datastore and SQL implementations.
   @Override
-  public <T> Optional<T> load(VKey<T> key) {
-    return Optional.of(getOfy().load().key(key.getOfyKey()).now());
+  public <T> Optional<T> maybeLoad(VKey<T> key) {
+    return Optional.ofNullable(getOfy().load().key(key.getOfyKey()).now());
+  }
+
+  @Override
+  public <T> T load(VKey<T> key) {
+    T result = getOfy().load().key(key.getOfyKey()).now();
+    if (result == null) {
+      throw new NoSuchElementException(key.toString());
+    }
+    return result;
   }
 
   @Override
   public <T> ImmutableList<T> load(Iterable<VKey<T>> keys) {
     Iterator<Key<T>> iter =
-        StreamSupport.stream(keys.spliterator(), false).map(key -> key.getOfyKey()).iterator();
+        StreamSupport.stream(keys.spliterator(), false).map(VKey::getOfyKey).iterator();
 
     // The lambda argument to keys() effectively converts Iterator -> Iterable.
     return ImmutableList.copyOf(getOfy().load().keys(() -> iter).values());
@@ -151,16 +166,23 @@ public class DatastoreTransactionManager implements TransactionManager {
 
   @Override
   public <T> ImmutableList<T> loadAll(Class<T> clazz) {
+    // We can do a ofy().load().type(clazz), but this doesn't work in a transaction.
     throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
   }
 
   @Override
-  public <T> int delete(VKey<T> key) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+  public void delete(VKey<?> key) {
+    getOfy().delete().key(key.getOfyKey()).now();
   }
 
   @Override
-  public <T> void assertDelete(VKey<T> key) {
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+  public void delete(Iterable<? extends VKey<?>> vKeys) {
+    // We have to create a list to work around the wildcard capture issue here.
+    // See https://docs.oracle.com/javase/tutorial/java/generics/capture.html
+    ImmutableList<Key<?>> list =
+        StreamSupport.stream(vKeys.spliterator(), false)
+            .map(VKey::getOfyKey)
+            .collect(toImmutableList());
+    getOfy().delete().keys(list).now();
   }
 }

core/src/main/java/google/registry/model/ofy/ObjectifyService.java

@@ -34,9 +34,9 @@ import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.impl.translate.TranslatorFactory;
 import com.googlecode.objectify.impl.translate.opt.joda.MoneyStringTranslatorFactory;
 import google.registry.config.RegistryEnvironment;
+import google.registry.model.Buildable;
 import google.registry.model.EntityClasses;
 import google.registry.model.ImmutableObject;
-import google.registry.model.host.HostResource;
 import google.registry.model.translators.BloomFilterOfStringTranslatorFactory;
 import google.registry.model.translators.CidrAddressBlockTranslatorFactory;
 import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
@@ -130,7 +130,7 @@ public class ObjectifyService {
             new InetAddressTranslatorFactory(),
             new MoneyStringTranslatorFactory(),
             new ReadableInstantUtcTranslatorFactory(),
-            new VKeyTranslatorFactory<HostResource>(HostResource.class),
+            new VKeyTranslatorFactory(),
             new UpdateAutoTimestampTranslatorFactory())) {
       factory().getTranslators().add(translatorFactory);
     }
@@ -151,11 +151,14 @@ public class ObjectifyService {
       String kind = Key.getKind(clazz);
       boolean registered = factory().getMetadata(kind) != null;
       if (clazz.isAnnotationPresent(Entity.class)) {
-        // Objectify silently ignores re-registrations for a given kind string, even if the classes
-        // being registered are distinct. Throw an exception if that would happen here.
-        checkState(!registered,
+        // Objectify silently replaces current registration for a given kind string when a different
+        // class is registered again for this kind. For simplicity's sake, throw an exception on any
+        // re-registration.
+        checkState(
+            !registered,
             "Kind '%s' already registered, cannot register new @Entity %s",
-            kind, clazz.getCanonicalName());
+            kind,
+            clazz.getCanonicalName());
       } else if (clazz.isAnnotationPresent(EntitySubclass.class)) {
         // Ensure that any @EntitySubclass classes have also had their parent @Entity registered,
         // which Objectify nominally requires but doesn't enforce in 4.x (though it may in 5.x).
@@ -165,10 +168,16 @@ public class ObjectifyService {
       }
       com.googlecode.objectify.ObjectifyService.register(clazz);
       // Autogenerated ids make the commit log code very difficult since we won't always be able
-      // to create a key for an entity immediately when requesting a save. Disallow that here.
-      checkState(
-          !factory().getMetadata(clazz).getKeyMetadata().isIdGeneratable(),
-          "Can't register %s: Autogenerated ids (@Id on a Long) are not supported.", kind);
+      // to create a key for an entity immediately when requesting a save. So, we require such
+      // entities to implement google.registry.model.Buildable as its build() function allocates the
+      // id to the entity.
+      if (factory().getMetadata(clazz).getKeyMetadata().isIdGeneratable()) {
+        checkState(
+            Buildable.class.isAssignableFrom(clazz),
+            "Can't register %s: Entity with autogenerated ids (@Id on a Long) must implement"
+                + " google.registry.model.Buildable.",
+            kind);
+      }
     }
   }