Refactor Fee handling so that each fee knows if it's premium (#626)

* Refactor Fee handling so that each fee knows if it's premium

This is a noop for now, as the new isPremium boolean isn't yet used by anything,
but it will be used in follow-up PRs to add additional fee information using the
fee extension (see: b/157621273).  Specifically what we're trying to do here is
return <fee:command name="create" standard="1"> (using the finalized version of
the fee extension) when an entire command has no premium fee associated with
it. And in the current earlier versions of the fee extension that we support,
we'll want to display the correct fee amount and class for creates/checks on
reserved domains when a valid allocation token is passed. This also needs the
isPremium information.

There are no testing implications yet because isPremium isn't exposed anywhere,
but there will definitely be lots of test changes once it's feeding into EPP-
visible changes.

* Rename things, add method Javadoc

* Apply formatting

README.md

@@ -33,7 +33,7 @@ running system:
 *   View the source code for the [GAE app](https://github.com/google/nomulus/tree/master/core/src/main/java/google/registry)
     and for the [GKE proxy](https://github.com/google/nomulus/tree/master/proxy/src/main/java/google/registry)
 *   [Other docs](https://github.com/google/nomulus/tree/master/docs)
-*   [Javadoc](https://nomulus.foo/javadoc/latest/)
+*   [Javadoc](https://javadoc.nomulus.foo/)
 *   [Nomulus discussion
     group](https://groups.google.com/forum/#!forum/nomulus-discuss), for any
     other questions

build.gradle

@@ -458,6 +458,7 @@ task javadoc(type: Javadoc) {
   source javadocSource
   classpath = files(javadocClasspath)
   destinationDir = file("${buildDir}/docs/javadoc")
+  options.encoding = "UTF-8"
   // In a lot of places we don't write @return so suppress warnings about that.
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
@@ -467,4 +468,4 @@ task javadoc(type: Javadoc) {
 
 tasks.build.dependsOn(tasks.javadoc)
 
-javadocDependentTasks.each { tasks.javadoc.shouldRunAfter(it) }
+javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }

core/src/main/java/google/registry/backup/AppEngineEnvironment.java

@@ -0,0 +1,66 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.apphosting.api.ApiProxy;
+import com.google.apphosting.api.ApiProxy.Environment;
+import com.google.common.collect.ImmutableMap;
+import java.io.Closeable;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+
+/**
+ * Sets up a placeholder {@link Environment} on a non-AppEngine platform so that Datastore Entities
+ * can be deserialized. See {@code DatastoreEntityExtension} in test source for more information.
+ */
+public class AppEngineEnvironment implements Closeable {
+
+  private static final Environment PLACEHOLDER_ENV = createAppEngineEnvironment();
+
+  private boolean isPlaceHolderNeeded;
+
+  AppEngineEnvironment() {
+    isPlaceHolderNeeded = ApiProxy.getCurrentEnvironment() == null;
+    // isPlaceHolderNeeded may be true when we are invoked in a test with AppEngineRule.
+    if (isPlaceHolderNeeded) {
+      ApiProxy.setEnvironmentForCurrentThread(PLACEHOLDER_ENV);
+    }
+  }
+
+  @Override
+  public void close() {
+    if (isPlaceHolderNeeded) {
+      ApiProxy.setEnvironmentForCurrentThread(null);
+    }
+  }
+
+  /** Returns a placeholder {@link Environment} that can return hardcoded AppId and Attributes. */
+  private static Environment createAppEngineEnvironment() {
+    return (Environment)
+        Proxy.newProxyInstance(
+            Environment.class.getClassLoader(),
+            new Class[] {Environment.class},
+            (Object proxy, Method method, Object[] args) -> {
+              switch (method.getName()) {
+                case "getAppId":
+                  return "PlaceholderAppId";
+                case "getAttributes":
+                  return ImmutableMap.<String, Object>of();
+                default:
+                  throw new UnsupportedOperationException(method.getName());
+              }
+            });
+  }
+}

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -0,0 +1,95 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.backup.BackupUtils.createDeserializingIterator;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Streams;
+import google.registry.model.ImmutableObject;
+import google.registry.model.ofy.CommitLogCheckpoint;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Iterator;
+import java.util.stream.Stream;
+
+/**
+ * Helpers for reading CommitLog records from a file.
+ *
+ * <p>This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial
+ * population of the Cloud SQL database.
+ */
+public final class CommitLogImports {
+
+  private CommitLogImports() {}
+
+  /**
+   * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link
+   * ImmutableList} of {@link VersionedEntity} records. Upon completion the {@code inputStream} is
+   * closed.
+   *
+   * <p>The returned list may be empty, since CommitLogs are written at fixed intervals regardless
+   * if actual changes exist.
+   *
+   * <p>A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated)
+   * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence
+   * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
+   * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
+   */
+  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
+        InputStream input = new BufferedInputStream(inputStream)) {
+      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
+      checkState(commitLogs.hasNext());
+      checkState(commitLogs.next() instanceof CommitLogCheckpoint);
+
+      return Streams.stream(commitLogs)
+          .map(
+              e ->
+                  e instanceof CommitLogManifest
+                      ? VersionedEntity.fromManifest((CommitLogManifest) e)
+                      : Stream.of(VersionedEntity.fromMutation((CommitLogMutation) e)))
+          .flatMap(s -> s)
+          .collect(ImmutableList.toImmutableList());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /** Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link File}. */
+  public static ImmutableList<VersionedEntity> loadEntities(File commitLogFile) {
+    try {
+      return loadEntities(new FileInputStream(commitLogFile));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link
+   * ReadableByteChannel}.
+   */
+  public static ImmutableList<VersionedEntity> loadEntities(ReadableByteChannel channel) {
+    return loadEntities(Channels.newInputStream(channel));
+  }
+}

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -0,0 +1,160 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.appengine.api.datastore.Key;
+import com.google.auto.value.AutoValue;
+import com.google.auto.value.extension.memoized.Memoized;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+
+/**
+ * A Datastore {@link Entity Entity's} timestamped state.
+ *
+ * <p>For a new or updated Entity, its ProtocolBuffer bytes are stored along with its {@link Key}.
+ * For a deleted entity, only its {@link Key} is stored, and the {@link #entityProtoBytes} is left
+ * as null.
+ *
+ * <p>Note that {@link Optional java.util.Optional} is not serializable, therefore cannot be used as
+ * property type in this class.
+ */
+@AutoValue
+public abstract class VersionedEntity implements Serializable {
+
+  private static final long serialVersionUID = 1L;
+
+  public abstract long commitTimeMills();
+
+  /** The {@link Key} of the {@link Entity}. */
+  public abstract Key key();
+
+  /** Serialized form of the {@link Entity}. This property is {@code null} for a deleted Entity. */
+  @Nullable
+  abstract ImmutableBytes entityProtoBytes();
+
+  @Memoized
+  public Optional<Entity> getEntity() {
+    return Optional.ofNullable(entityProtoBytes())
+        .map(ImmutableBytes::getBytes)
+        .map(EntityTranslator::createFromPbBytes);
+  }
+
+  public boolean isDelete() {
+    return entityProtoBytes() == null;
+  }
+
+  /**
+   * Converts deleted entity keys in {@code manifest} into a {@link Stream} of {@link
+   * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
+   * information.
+   */
+  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+    long commitTimeMillis = manifest.getCommitTime().getMillis();
+    return manifest.getDeletions().stream()
+        .map(com.googlecode.objectify.Key::getRaw)
+        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+  }
+
+  /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
+  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+    return from(
+        com.googlecode.objectify.Key.create(mutation).getParent().getId(),
+        mutation.getEntityProtoBytes());
+  }
+
+  public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
+    return builder()
+        .entityProtoBytes(entityProtoBytes)
+        .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
+        .commitTimeMills(commitTimeMillis)
+        .build();
+  }
+
+  static Builder builder() {
+    return new AutoValue_VersionedEntity.Builder();
+  }
+
+  @AutoValue.Builder
+  public abstract static class Builder {
+
+    public abstract Builder commitTimeMills(long commitTimeMillis);
+
+    abstract Builder entityProtoBytes(ImmutableBytes bytes);
+
+    public abstract Builder key(Key key);
+
+    public abstract VersionedEntity build();
+
+    public Builder entityProtoBytes(byte[] bytes) {
+      return entityProtoBytes(new ImmutableBytes(bytes));
+    }
+  }
+
+  /**
+   * Wraps a byte array and prevents it from being modified by its original owner.
+   *
+   * <p>While this class seems an overkill, it exists for two reasons:
+   *
+   * <ul>
+   *   <li>It is easier to override the {@link #equals} method here (for value-equivalence check)
+   *       than to override the AutoValue-generated {@code equals} method.
+   *   <li>To appease the style checker, which forbids arrays as AutoValue property.
+   * </ul>
+   */
+  static final class ImmutableBytes implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private final byte[] bytes;
+
+    ImmutableBytes(byte[] bytes) {
+      this.bytes = Arrays.copyOf(bytes, bytes.length);
+    }
+
+    /**
+     * Returns the saved byte array. Invocation is restricted to trusted callers, who must not
+     * modify the array.
+     */
+    byte[] getBytes() {
+      return bytes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof ImmutableBytes)) {
+        return false;
+      }
+      ImmutableBytes that = (ImmutableBytes) o;
+      // Do not use Objects.equals, which checks reference identity instead of data in array.
+      return Arrays.equals(bytes, that.bytes);
+    }
+
+    @Override
+    public int hashCode() {
+      // Do not use Objects.hashCode, which hashes the reference, not the data in array.
+      return Arrays.hashCode(bytes);
+    }
+  }
+}

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -150,7 +150,7 @@ public final class AsyncTaskEnqueuer {
 
   /** Enqueues a task to asynchronously refresh DNS for a renamed host. */
   public void enqueueAsyncDnsRefresh(HostResource host, DateTime now) {
-    VKey<HostResource> hostKey = host.createKey();
+    VKey<HostResource> hostKey = host.createVKey();
     logger.atInfo().log("Enqueuing async DNS refresh for renamed host %s.", hostKey);
     addTaskToQueueWithRetry(
         asyncDnsRefreshPullQueue,

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -347,7 +347,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
       String resourceClientId = resource.getPersistedCurrentSponsorClientId();
       if (resource instanceof HostResource && ((HostResource) resource).isSubordinate()) {
         resourceClientId =
-            ofy().load().key(((HostResource) resource).getSuperordinateDomain()).now()
+            tm().load(((HostResource) resource).getSuperordinateDomain())
                 .cloneProjectedAtTime(now)
                 .getCurrentSponsorClientId();
       }
@@ -466,10 +466,11 @@ public class DeleteContactsAndHostsAction implements Runnable {
         HostResource host = (HostResource) existingResource;
         if (host.isSubordinate()) {
           dnsQueue.addHostRefreshTask(host.getFullyQualifiedHostName());
-          ofy().save().entity(
-              ofy().load().key(host.getSuperordinateDomain()).now().asBuilder()
-                  .removeSubordinateHost(host.getFullyQualifiedHostName())
-                  .build());
+          tm().saveNewOrUpdate(
+                  tm().load(host.getSuperordinateDomain())
+                      .asBuilder()
+                      .removeSubordinateHost(host.getFullyQualifiedHostName())
+                      .build());
         }
       } else {
         throw new IllegalStateException(

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -0,0 +1,74 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import org.joda.time.DateTime;
+
+/**
+ * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a
+ * Datastore export and Nomulus CommitLogs that overlap with the export.
+ */
+public final class BackupPaths {
+
+  private BackupPaths() {}
+
+  private static final String WILDCARD_CHAR = "*";
+  private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/input-%s";
+
+  public static final String COMMIT_LOG_NAME_PREFIX = "commit_diff_until_";
+  private static final String COMMIT_LOG_PATTERN_TEMPLATE = "%s/" + COMMIT_LOG_NAME_PREFIX + "*";
+
+  /**
+   * Returns a regex pattern that matches all Datastore export files of a given {@code kind}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kind the 'kind' of the Datastore entity
+   */
+  public static String getExportFileNamePattern(String exportDir, String kind) {
+    checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
+    checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, WILDCARD_CHAR);
+  }
+
+  /**
+   * Returns the fully qualified path of a Datastore export file with the given {@code kind} and
+   * {@code shard}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kind the 'kind' of the Datastore entity
+   * @param shard an integer suffix of the file name
+   */
+  public static String getExportFileNameByShard(String exportDir, String kind, int shard) {
+    checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
+    checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
+    checkArgument(shard >= 0, "Negative shard %s not allowed.", shard);
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, Integer.toString(shard));
+  }
+
+  public static String getCommitLogFileNamePattern(String commitLogDir) {
+    return String.format(COMMIT_LOG_PATTERN_TEMPLATE, commitLogDir);
+  }
+
+  /** Gets the Commit timestamp from a CommitLog file name. */
+  public static DateTime getCommitLogTimestamp(String fileName) {
+    checkArgument(!isNullOrEmpty(fileName), "Null or empty fileName.");
+    int start = fileName.lastIndexOf(COMMIT_LOG_NAME_PREFIX);
+    checkArgument(start >= 0, "Illegal file name %s.", fileName);
+    return DateTime.parse(fileName.substring(start + COMMIT_LOG_NAME_PREFIX.length()));
+  }
+}

core/src/main/java/google/registry/beam/initsql/CommitLogTransforms.java

@@ -0,0 +1,109 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static google.registry.beam.initsql.BackupPaths.getCommitLogFileNamePattern;
+import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
+import static google.registry.beam.initsql.Transforms.processFiles;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+
+import google.registry.backup.CommitLogImports;
+import google.registry.backup.VersionedEntity;
+import java.io.IOException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.DateTime;
+
+/**
+ * {@link org.apache.beam.sdk.transforms.PTransform Pipeline transforms} for loading from Nomulus
+ * CommitLog files. They are all part of a transformation that loads raw records from a sequence of
+ * Datastore CommitLog files, and are broken apart for testing.
+ */
+public class CommitLogTransforms {
+
+  /**
+   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
+   * all Datastore CommitLog files.
+   */
+  public static PTransform<PBegin, PCollection<String>> getCommitLogFilePatterns(
+      String commitLogDir) {
+    return Create.of(getCommitLogFileNamePattern(commitLogDir)).withCoder(StringUtf8Coder.of());
+  }
+
+  /**
+   * Returns files with timestamps between {@code fromTime} (inclusive) and {@code endTime}
+   * (exclusive).
+   */
+  public static PTransform<PCollection<? extends String>, PCollection<String>>
+      filterCommitLogsByTime(DateTime fromTime, DateTime toTime) {
+    checkNotNull(fromTime, "fromTime");
+    checkNotNull(toTime, "toTime");
+    checkArgument(
+        fromTime.isBefore(toTime),
+        "Invalid time range: fromTime (%s) is before endTime (%s)",
+        fromTime,
+        toTime);
+    return ParDo.of(new FilterCommitLogFileByTime(fromTime, toTime));
+  }
+
+  /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
+  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
+      loadCommitLogsFromFiles() {
+    return processFiles(new LoadOneCommitLogsFile());
+  }
+
+  static class FilterCommitLogFileByTime extends DoFn<String, String> {
+    private final DateTime fromTime;
+    private final DateTime toTime;
+
+    public FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
+      this.fromTime = fromTime;
+      this.toTime = toTime;
+    }
+
+    @ProcessElement
+    public void processElement(@Element String fileName, OutputReceiver<String> out) {
+      DateTime timestamp = getCommitLogTimestamp(fileName);
+      if (isBeforeOrAt(fromTime, timestamp) && timestamp.isBefore(toTime)) {
+        out.output(fileName);
+      }
+    }
+  }
+
+  /**
+   * Reads a CommitLog file and converts its content into {@link VersionedEntity VersionedEntities}.
+   */
+  static class LoadOneCommitLogsFile extends DoFn<ReadableFile, VersionedEntity> {
+
+    @ProcessElement
+    public void processElement(@Element ReadableFile file, OutputReceiver<VersionedEntity> out) {
+      try {
+        CommitLogImports.loadEntities(file.open()).forEach(out::output);
+      } catch (IOException e) {
+        // Let the pipeline retry the whole file.
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

core/src/main/java/google/registry/beam/initsql/ExportLoadingTransforms.java

@@ -0,0 +1,82 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static google.registry.beam.initsql.BackupPaths.getExportFileNamePattern;
+import static google.registry.beam.initsql.Transforms.processFiles;
+
+import com.google.common.collect.ImmutableList;
+import google.registry.backup.VersionedEntity;
+import google.registry.tools.LevelDbLogReader;
+import java.io.IOException;
+import java.util.Collection;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * {@link PTransform Pipeline transforms} for loading from Datastore export files. They are all part
+ * of a transformation that loads raw records from a Datastore export, and are broken apart for
+ * testing.
+ */
+public class ExportLoadingTransforms {
+
+  /**
+   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
+   * all Datastore export files of the given {@code kinds}.
+   */
+  public static PTransform<PBegin, PCollection<String>> getDatastoreExportFilePatterns(
+      String exportDir, Collection<String> kinds) {
+    return Create.of(
+            kinds.stream()
+                .map(kind -> getExportFileNamePattern(exportDir, kind))
+                .collect(ImmutableList.toImmutableList()))
+        .withCoder(StringUtf8Coder.of());
+  }
+
+  /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
+  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
+      loadExportDataFromFiles() {
+    return processFiles(new LoadOneExportShard());
+  }
+
+  /**
+   * Reads a LevelDb file and converts each raw record into a {@link VersionedEntity}. All such
+   * entities use {@link Long#MIN_VALUE} as timestamp, so that they go before data from CommitLogs.
+   *
+   * <p>LevelDb files are not seekable because a large object may span multiple blocks. If a
+   * sequential read fails, the file needs to be retried from the beginning.
+   */
+  private static class LoadOneExportShard extends DoFn<ReadableFile, VersionedEntity> {
+
+    private static final long TIMESTAMP = Long.MIN_VALUE;
+
+    @ProcessElement
+    public void processElement(@Element ReadableFile file, OutputReceiver<VersionedEntity> output) {
+      try {
+        LevelDbLogReader.from(file.open())
+            .forEachRemaining(record -> output.output(VersionedEntity.from(TIMESTAMP, record)));
+      } catch (IOException e) {
+        // Let the pipeline retry the whole file.
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

core/src/main/java/google/registry/beam/initsql/README.md

@@ -0,0 +1,3 @@
+## Summary
+
+This package contains a BEAM pipeline that populates a Cloud SQL database from a Datastore backup.

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -0,0 +1,61 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import google.registry.backup.VersionedEntity;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Common {@link PTransform pipeline transforms} used in pipelines that load from both Datastore
+ * export files and Nomulus CommitLog files.
+ */
+public class Transforms {
+
+  /**
+   * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}.
+   */
+  public static PTransform<PCollection<String>, PCollection<Metadata>> getFilesByPatterns() {
+    return new PTransform<PCollection<String>, PCollection<Metadata>>() {
+      @Override
+      public PCollection<Metadata> expand(PCollection<String> input) {
+        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+      }
+    };
+  }
+
+  /**
+   * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using
+   * caller-provided {@code transformer}.
+   */
+  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
+      DoFn<ReadableFile, VersionedEntity> transformer) {
+    return new PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>() {
+      @Override
+      public PCollection<VersionedEntity> expand(PCollection<Metadata> input) {
+        return input
+            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+            .apply(transformer.getClass().getSimpleName(), ParDo.of(transformer));
+      }
+    };
+  }
+}

core/src/main/java/google/registry/beam/spec11/ThreatMatch.java

@@ -24,22 +24,10 @@ import org.json.JSONObject;
 public abstract class ThreatMatch implements Serializable {
 
   private static final String THREAT_TYPE_FIELD = "threatType";
-  private static final String PLATFORM_TYPE_FIELD = "platformType";
-  private static final String METADATA_FIELD = "threatEntryMetadata";
   private static final String DOMAIN_NAME_FIELD = "fullyQualifiedDomainName";
 
   /** Returns what kind of threat it is (malware, phishing etc.) */
   public abstract String threatType();
-  /** Returns what platforms it affects (Windows, Linux etc.) */
-  abstract String platformType();
-  /**
-   * Returns a String representing a JSON Object containing arbitrary metadata associated with this
-   * threat, or "NONE" if there is no metadata to retrieve.
-   *
-   * <p>This ideally would be a {@link JSONObject} type, but can't be due to serialization
-   * requirements.
-   */
-  abstract String metadata();
   /** Returns the fully qualified domain name [SLD].[TLD] of the matched threat. */
   public abstract String fullyQualifiedDomainName();
 
@@ -52,29 +40,19 @@ public abstract class ThreatMatch implements Serializable {
   static ThreatMatch create(JSONObject threatMatchJSON, String fullyQualifiedDomainName)
       throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatchJSON.getString(THREAT_TYPE_FIELD),
-        threatMatchJSON.getString(PLATFORM_TYPE_FIELD),
-        threatMatchJSON.has(METADATA_FIELD)
-            ? threatMatchJSON.getJSONObject(METADATA_FIELD).toString()
-            : "NONE",
-        fullyQualifiedDomainName);
+        threatMatchJSON.getString(THREAT_TYPE_FIELD), fullyQualifiedDomainName);
   }
 
   /** Returns a {@link JSONObject} representing a subset of this object's data. */
   JSONObject toJSON() throws JSONException {
     return new JSONObject()
         .put(THREAT_TYPE_FIELD, threatType())
-        .put(PLATFORM_TYPE_FIELD, platformType())
-        .put(METADATA_FIELD, metadata())
         .put(DOMAIN_NAME_FIELD, fullyQualifiedDomainName());
   }
 
   /** Parses a {@link JSONObject} and returns an equivalent {@link ThreatMatch}. */
   public static ThreatMatch fromJSON(JSONObject threatMatch) throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatch.getString(THREAT_TYPE_FIELD),
-        threatMatch.getString(PLATFORM_TYPE_FIELD),
-        threatMatch.getString(METADATA_FIELD),
-        threatMatch.getString(DOMAIN_NAME_FIELD));
+        threatMatch.getString(THREAT_TYPE_FIELD), threatMatch.getString(DOMAIN_NAME_FIELD));
   }
 }

core/src/main/java/google/registry/flows/ResourceFlowUtils.java

@@ -94,10 +94,10 @@ public final class ResourceFlowUtils {
                    * trust the query and need to do the full mapreduce.
                    */
                   Iterable<Key<DomainBase>> keys =
-                      queryForLinkedDomains(fki.getResourceKey(), now)
+                      queryForLinkedDomains(fki.getResourceKey().getOfyKey(), now)
                           .limit(FAILFAST_CHECK_COUNT)
                           .keys();
-                  VKey<R> resourceVKey = VKey.createOfy(resourceClass, fki.getResourceKey());
+                  VKey<R> resourceVKey = fki.getResourceKey();
                   Predicate<DomainBase> predicate =
                       domain -> getPotentialReferences.apply(domain).contains(resourceVKey);
                   return ofy().load().keys(keys).values().stream().anyMatch(predicate)
@@ -136,9 +136,9 @@ public final class ResourceFlowUtils {
 
   public static <R extends EppResource> void verifyResourceDoesNotExist(
       Class<R> clazz, String targetId, DateTime now, String clientId) throws EppException {
-    Key<R> key = loadAndGetKey(clazz, targetId, now);
+    VKey<R> key = loadAndGetKey(clazz, targetId, now);
     if (key != null) {
-      R resource = ofy().load().key(key).now();
+      R resource = tm().load(key);
       // These are similar exceptions, but we can track them internally as log-based metrics.
       if (Objects.equals(clientId, resource.getPersistedCurrentSponsorClientId())) {
         throw new ResourceAlreadyExistsForThisClientException(targetId);

core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java

@@ -97,7 +97,7 @@ public final class ContactTransferApproveFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, gainingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java

@@ -93,7 +93,7 @@ public final class ContactTransferCancelFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, losingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java

@@ -90,7 +90,7 @@ public final class ContactTransferRejectFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, gainingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java

@@ -126,13 +126,15 @@ public final class ContactTransferRequestFlow implements TransactionalFlow {
     // If the transfer is server approved, this message will be sent to the gaining registrar. */
     PollMessage serverApproveGainingPollMessage =
         createGainingTransferPollMessage(targetId, serverApproveTransferData, historyEntry);
-    TransferData pendingTransferData = serverApproveTransferData.asBuilder()
-        .setTransferStatus(TransferStatus.PENDING)
-        .setServerApproveEntities(
-            ImmutableSet.of(
-                Key.create(serverApproveGainingPollMessage),
-                Key.create(serverApproveLosingPollMessage)))
-        .build();
+    TransferData pendingTransferData =
+        serverApproveTransferData
+            .asBuilder()
+            .setTransferStatus(TransferStatus.PENDING)
+            .setServerApproveEntities(
+                ImmutableSet.of(
+                    serverApproveGainingPollMessage.createVKey(),
+                    serverApproveLosingPollMessage.createVKey()))
+            .build();
     // When a transfer is requested, a poll message is created to notify the losing registrar.
     PollMessage requestPollMessage =
         createLosingTransferPollMessage(targetId, pendingTransferData, historyEntry).asBuilder()

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -77,7 +77,6 @@ import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.billing.BillingEvent.Recurring;
-import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainCommand;
 import google.registry.model.domain.DomainCommand.Create;
@@ -353,14 +352,12 @@ public class DomainCreateFlow implements TransactionalFlow {
             .setLaunchNotice(hasClaimsNotice ? launchCreate.get().getNotice() : null)
             .setSmdId(signedMarkId)
             .setDsData(secDnsCreate.isPresent() ? secDnsCreate.get().getDsData() : null)
-            .setRegistrant(VKey.createOfy(ContactResource.class, command.getRegistrant()))
+            .setRegistrant(command.getRegistrant())
             .setAuthInfo(command.getAuthInfo())
             .setFullyQualifiedDomainName(targetId)
             .setNameservers(
                 (ImmutableSet<VKey<HostResource>>)
-                    command.getNameservers().stream()
-                        .map(key -> VKey.createOfy(HostResource.class, key))
-                        .collect(toImmutableSet()))
+                    command.getNameservers().stream().collect(toImmutableSet()))
             .setStatusValues(statuses.build())
             .setContacts(command.getContacts())
             .addGracePeriod(GracePeriod.forBillingEvent(GracePeriodStatus.ADD, createBillingEvent))

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -307,14 +307,11 @@ public class DomainFlowUtils {
   /** Verify that no linked resources have disallowed statuses. */
   static void verifyNotInPendingDelete(
       Set<DesignatedContact> contacts,
-      Key<ContactResource> registrant,
-      Set<Key<HostResource>> nameservers)
+      VKey<ContactResource> registrant,
+      Set<VKey<HostResource>> nameservers)
       throws EppException {
-    ImmutableList.Builder<Key<? extends EppResource>> keysToLoad = new ImmutableList.Builder<>();
-    contacts.stream()
-        .map(DesignatedContact::getContactKey)
-        .map(VKey::getOfyKey)
-        .forEach(keysToLoad::add);
+    ImmutableList.Builder<VKey<? extends EppResource>> keysToLoad = new ImmutableList.Builder<>();
+    contacts.stream().map(DesignatedContact::getContactKey).forEach(keysToLoad::add);
     Optional.ofNullable(registrant).ifPresent(keysToLoad::add);
     keysToLoad.addAll(nameservers);
     verifyNotInPendingDelete(EppResource.loadCached(keysToLoad.build()).values());
@@ -381,7 +378,7 @@ public class DomainFlowUtils {
   }
 
   static void validateRequiredContactsPresent(
-      @Nullable Key<ContactResource> registrant, Set<DesignatedContact> contacts)
+      @Nullable VKey<ContactResource> registrant, Set<DesignatedContact> contacts)
       throws RequiredParameterMissingException {
     if (registrant == null) {
       throw new MissingRegistrantException();
@@ -693,7 +690,7 @@ public class DomainFlowUtils {
     List<Fee> fees = feeCommand.get().getFees();
     // The schema guarantees that at least one fee will be present.
     checkState(!fees.isEmpty());
-    BigDecimal total = BigDecimal.ZERO;
+    BigDecimal total = zeroInCurrency(feeCommand.get().getCurrency());
     for (Fee fee : fees) {
       if (!fee.hasDefaultAttributes()) {
         throw new UnsupportedFeeAttributeException();
@@ -941,6 +938,16 @@ public class DomainFlowUtils {
     }
   }
 
+  /**
+   * Returns zero for a specific currency.
+   *
+   * <p>{@link BigDecimal} has a concept of significant figures, so zero is not always zero. E.g.
+   * zero in USD is 0.00, whereas zero in Yen is 0, and zero in Dinars is 0.000 (!).
+   */
+  static BigDecimal zeroInCurrency(CurrencyUnit currencyUnit) {
+    return Money.of(currencyUnit, BigDecimal.ZERO).getAmount();
+  }
+
   /**
    * Check that if there's a claims notice it's on the claims list, and that if there's not one it's
    * not on the claims list.

core/src/main/java/google/registry/flows/domain/DomainPricingLogic.java

@@ -14,8 +14,9 @@
 
 package google.registry.flows.domain;
 
+import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
 import static google.registry.pricing.PricingEngineProxy.getDomainFeeClass;
-import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
+import static google.registry.pricing.PricingEngineProxy.getPricesForDomainName;
 
 import com.google.common.net.InternetDomainName;
 import google.registry.flows.EppException;
@@ -33,7 +34,6 @@ import google.registry.model.domain.fee.Fee;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.pricing.PremiumPricingEngine.DomainPrices;
 import google.registry.model.registry.Registry;
-import google.registry.pricing.PricingEngineProxy;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.util.Optional;
@@ -64,21 +64,27 @@ public final class DomainPricingLogic {
   public FeesAndCredits getCreatePrice(
       Registry registry,
       String domainName,
-      DateTime date,
+      DateTime dateTime,
       int years,
       boolean isAnchorTenant,
       Optional<AllocationToken> allocationToken)
       throws EppException {
     CurrencyUnit currency = registry.getCurrency();
+
+    BaseFee createFeeOrCredit;
     // Domain create cost is always zero for anchor tenants
-    Money domainCreateCost =
-        isAnchorTenant
-            ? Money.of(currency, BigDecimal.ZERO)
-            : getDomainCreateCostWithDiscount(domainName, date, years, allocationToken);
-    BaseFee createFeeOrCredit = Fee.create(domainCreateCost.getAmount(), FeeType.CREATE);
+    if (isAnchorTenant) {
+      createFeeOrCredit = Fee.create(zeroInCurrency(currency), FeeType.CREATE, false);
+    } else {
+      DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
+      Money domainCreateCost =
+          getDomainCreateCostWithDiscount(domainPrices, years, allocationToken);
+      createFeeOrCredit =
+          Fee.create(domainCreateCost.getAmount(), FeeType.CREATE, domainPrices.isPremium());
+    }
 
     // Create fees for the cost and the EAP fee, if any.
-    Fee eapFee = registry.getEapFeeFor(date);
+    Fee eapFee = registry.getEapFeeFor(dateTime);
     FeesAndCredits.Builder feesBuilder =
         new FeesAndCredits.Builder().setCurrency(currency).addFeeOrCredit(createFeeOrCredit);
     // Don't charge anchor tenants EAP fees.
@@ -92,7 +98,7 @@ public final class DomainPricingLogic {
             .setFeesAndCredits(feesBuilder.build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .setYears(years)
             .build());
   }
@@ -100,69 +106,73 @@ public final class DomainPricingLogic {
   /** Returns a new renew price for the pricer. */
   @SuppressWarnings("unused")
   public FeesAndCredits getRenewPrice(
-      Registry registry,
-      String domainName,
-      DateTime date,
-      int years)
-      throws EppException {
-    Money renewCost = getDomainRenewCost(domainName, date, years);
+      Registry registry, String domainName, DateTime dateTime, int years) throws EppException {
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
+    BigDecimal renewCost = domainPrices.getRenewCost().multipliedBy(years).getAmount();
     return customLogic.customizeRenewPrice(
         RenewPriceParameters.newBuilder()
             .setFeesAndCredits(
                 new FeesAndCredits.Builder()
                     .setCurrency(registry.getCurrency())
-                    .addFeeOrCredit(Fee.create(renewCost.getAmount(), FeeType.RENEW))
+                    .addFeeOrCredit(Fee.create(renewCost, FeeType.RENEW, domainPrices.isPremium()))
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .setYears(years)
             .build());
   }
 
   /** Returns a new restore price for the pricer. */
-  public FeesAndCredits getRestorePrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getRestorePrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
     FeesAndCredits feesAndCredits =
         new FeesAndCredits.Builder()
             .setCurrency(registry.getCurrency())
             .addFeeOrCredit(
-                Fee.create(getDomainRenewCost(domainName, date, 1).getAmount(), FeeType.RENEW))
+                Fee.create(
+                    domainPrices.getRenewCost().getAmount(),
+                    FeeType.RENEW,
+                    domainPrices.isPremium()))
             .addFeeOrCredit(
-                Fee.create(registry.getStandardRestoreCost().getAmount(), FeeType.RESTORE))
+                Fee.create(registry.getStandardRestoreCost().getAmount(), FeeType.RESTORE, false))
             .build();
     return customLogic.customizeRestorePrice(
         RestorePriceParameters.newBuilder()
             .setFeesAndCredits(feesAndCredits)
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns a new transfer price for the pricer. */
-  public FeesAndCredits getTransferPrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getTransferPrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
-    Money renewCost = getDomainRenewCost(domainName, date, 1);
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
     return customLogic.customizeTransferPrice(
         TransferPriceParameters.newBuilder()
             .setFeesAndCredits(
                 new FeesAndCredits.Builder()
                     .setCurrency(registry.getCurrency())
-                    .addFeeOrCredit(Fee.create(renewCost.getAmount(), FeeType.RENEW))
+                    .addFeeOrCredit(
+                        Fee.create(
+                            domainPrices.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            domainPrices.isPremium()))
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns a new update price for the pricer. */
-  public FeesAndCredits getUpdatePrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getUpdatePrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
     CurrencyUnit currency = registry.getCurrency();
-    BaseFee feeOrCredit =
-        Fee.create(Money.zero(registry.getCurrency()).getAmount(), FeeType.UPDATE);
+    BaseFee feeOrCredit = Fee.create(zeroInCurrency(currency), FeeType.UPDATE, false);
     return customLogic.customizeUpdatePrice(
         UpdatePriceParameters.newBuilder()
             .setFeesAndCredits(
@@ -172,19 +182,19 @@ public final class DomainPricingLogic {
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns the fee class for a given domain and date. */
-  public Optional<String> getFeeClass(String domainName, DateTime date) {
-    return getDomainFeeClass(domainName, date);
+  public Optional<String> getFeeClass(String domainName, DateTime dateTime) {
+    return getDomainFeeClass(domainName, dateTime);
   }
 
+  /** Returns the domain create cost with allocation-token-related discounts applied. */
   private Money getDomainCreateCostWithDiscount(
-      String domainName, DateTime date, int years, Optional<AllocationToken> allocationToken)
+      DomainPrices domainPrices, int years, Optional<AllocationToken> allocationToken)
       throws EppException {
-    DomainPrices domainPrices = PricingEngineProxy.getPricesForDomainName(domainName, date);
     if (allocationToken.isPresent()
         && allocationToken.get().getDiscountFraction() != 0.0
         && domainPrices.isPremium()) {

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -211,8 +211,7 @@ public final class DomainRenewFlow implements TransactionalFlow {
             BeforeResponseParameters.newBuilder()
                 .setDomain(newDomain)
                 .setResData(DomainRenewData.create(targetId, newExpirationTime))
-                .setResponseExtensions(
-                    createResponseExtensions(feesAndCredits.getTotalCost(), feeRenew))
+                .setResponseExtensions(createResponseExtensions(feesAndCredits, feeRenew))
                 .build());
     return responseBuilder
         .setResData(responseData.resData())
@@ -270,14 +269,19 @@ public final class DomainRenewFlow implements TransactionalFlow {
   }
 
   private ImmutableList<FeeTransformResponseExtension> createResponseExtensions(
-      Money renewCost, Optional<FeeRenewCommandExtension> feeRenew) {
+      FeesAndCredits feesAndCredits, Optional<FeeRenewCommandExtension> feeRenew) {
     return feeRenew.isPresent()
         ? ImmutableList.of(
             feeRenew
                 .get()
                 .createResponseBuilder()
-                .setCurrency(renewCost.getCurrencyUnit())
-                .setFees(ImmutableList.of(Fee.create(renewCost.getAmount(), FeeType.RENEW)))
+                .setCurrency(feesAndCredits.getCurrency())
+                .setFees(
+                    ImmutableList.of(
+                        Fee.create(
+                            feesAndCredits.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RENEW))))
                 .build())
         : ImmutableList.of();
   }

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -143,24 +143,30 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     verifyRestoreAllowed(command, existingDomain, feeUpdate, feesAndCredits, now);
     HistoryEntry historyEntry = buildHistoryEntry(existingDomain, now);
     ImmutableSet.Builder<ImmutableObject> entitiesToSave = new ImmutableSet.Builder<>();
-    entitiesToSave.addAll(
-        createRestoreAndRenewBillingEvents(
-            historyEntry, feesAndCredits.getRestoreCost(), feesAndCredits.getRenewCost(), now));
-    // We don't preserve the original expiration time of the domain when we restore, since doing so
-    // would require us to know if they received a grace period refund when they deleted the domain,
-    // and to charge them for that again. Instead, we just say that all restores get a fresh year of
-    // registration and bill them for that accordingly.
-    DateTime newExpirationTime = now.plusYears(1);
-    BillingEvent.Recurring autorenewEvent = newAutorenewBillingEvent(existingDomain)
-        .setEventTime(newExpirationTime)
-        .setRecurrenceEndTime(END_OF_TIME)
-        .setParent(historyEntry)
-        .build();
-    PollMessage.Autorenew autorenewPollMessage = newAutorenewPollMessage(existingDomain)
-        .setEventTime(newExpirationTime)
-        .setAutorenewEndTime(END_OF_TIME)
-        .setParent(historyEntry)
-        .build();
+
+    // Restore the expiration time on the deleted domain, except if that's already passed, then add
+    // a year and bill for it immediately, with no grace period.
+    DateTime newExpirationTime = existingDomain.getRegistrationExpirationTime();
+    if (newExpirationTime.isBefore(now)) {
+      entitiesToSave.add(createRenewBillingEvent(historyEntry, feesAndCredits.getRenewCost(), now));
+      newExpirationTime = newExpirationTime.plusYears(1);
+    }
+    // Always bill for the restore itself.
+    entitiesToSave.add(
+        createRestoreBillingEvent(historyEntry, feesAndCredits.getRestoreCost(), now));
+
+    BillingEvent.Recurring autorenewEvent =
+        newAutorenewBillingEvent(existingDomain)
+            .setEventTime(newExpirationTime)
+            .setRecurrenceEndTime(END_OF_TIME)
+            .setParent(historyEntry)
+            .build();
+    PollMessage.Autorenew autorenewPollMessage =
+        newAutorenewPollMessage(existingDomain)
+            .setEventTime(newExpirationTime)
+            .setAutorenewEndTime(END_OF_TIME)
+            .setParent(historyEntry)
+            .build();
     DomainBase newDomain =
         performRestore(
             existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId);
@@ -170,9 +176,7 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     ofy().delete().key(existingDomain.getDeletePollMessage());
     dnsQueue.addDomainRefreshTask(existingDomain.getFullyQualifiedDomainName());
     return responseBuilder
-        .setExtensions(
-            createResponseExtensions(
-                feesAndCredits.getRestoreCost(), feesAndCredits.getRenewCost(), feeUpdate))
+        .setExtensions(createResponseExtensions(feesAndCredits, feeUpdate))
         .build();
   }
 
@@ -212,18 +216,6 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     validateFeeChallenge(targetId, now, feeUpdate, feesAndCredits);
   }
 
-  private ImmutableSet<BillingEvent.OneTime> createRestoreAndRenewBillingEvents(
-      HistoryEntry historyEntry, Money restoreCost, Money renewCost, DateTime now) {
-    // Bill for the restore.
-    BillingEvent.OneTime restoreEvent = createRestoreBillingEvent(historyEntry, restoreCost, now);
-    // Create a new autorenew billing event and poll message starting at the new expiration time.
-    // Also bill for the 1 year cost of a domain renew. This is to avoid registrants being able to
-    // game the system for premium names by renewing, deleting, and then restoring to get a free
-    // year. Note that this billing event has no grace period; it is effective immediately.
-    BillingEvent.OneTime renewEvent = createRenewBillingEvent(historyEntry, renewCost, now);
-    return ImmutableSet.of(restoreEvent, renewEvent);
-  }
-
   private static DomainBase performRestore(
       DomainBase existingDomain,
       DateTime newExpirationTime,
@@ -271,17 +263,23 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
   }
 
   private static ImmutableList<FeeTransformResponseExtension> createResponseExtensions(
-      Money restoreCost, Money renewCost, Optional<FeeUpdateCommandExtension> feeUpdate) {
+      FeesAndCredits feesAndCredits, Optional<FeeUpdateCommandExtension> feeUpdate) {
     return feeUpdate.isPresent()
         ? ImmutableList.of(
             feeUpdate
                 .get()
                 .createResponseBuilder()
-                .setCurrency(restoreCost.getCurrencyUnit())
+                .setCurrency(feesAndCredits.getCurrency())
                 .setFees(
                     ImmutableList.of(
-                        Fee.create(restoreCost.getAmount(), FeeType.RESTORE),
-                        Fee.create(renewCost.getAmount(), FeeType.RENEW)))
+                        Fee.create(
+                            feesAndCredits.getRestoreCost().getAmount(),
+                            FeeType.RESTORE,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RESTORE)),
+                        Fee.create(
+                            feesAndCredits.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RENEW))))
                 .build())
         : ImmutableList.of();
   }

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -214,7 +214,7 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
     ofy().save().entities(entitiesToSave.build());
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(
             targetId, newDomain.getTransferData(), newDomain.getRegistrationExpirationTime()))

core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java

@@ -110,7 +110,7 @@ public final class DomainTransferCancelFlow implements TransactionalFlow {
     updateAutorenewRecurrenceEndTime(existingDomain, END_OF_TIME);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newDomain.getTransferData(), null))
         .build();

core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java

@@ -112,7 +112,7 @@ public final class DomainTransferRejectFlow implements TransactionalFlow {
     updateAutorenewRecurrenceEndTime(existingDomain, END_OF_TIME);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newDomain.getTransferData(), null))
         .build();

core/src/main/java/google/registry/flows/domain/DomainTransferUtils.java

@@ -20,7 +20,6 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
@@ -37,6 +36,7 @@ import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import org.joda.money.Money;
@@ -52,37 +52,34 @@ public final class DomainTransferUtils {
       TransferData.Builder transferDataBuilder,
       ImmutableSet<TransferServerApproveEntity> serverApproveEntities,
       Period transferPeriod) {
-    ImmutableSet.Builder<Key<? extends TransferServerApproveEntity>> serverApproveEntityKeys =
+    ImmutableSet.Builder<VKey<? extends TransferServerApproveEntity>> serverApproveEntityKeys =
         new ImmutableSet.Builder<>();
     for (TransferServerApproveEntity entity : serverApproveEntities) {
-      serverApproveEntityKeys.add(Key.create(entity));
+      serverApproveEntityKeys.add(entity.createVKey());
     }
     if (transferPeriod.getValue() != 0) {
       // Unless superuser sets period to 0, add a transfer billing event.
       transferDataBuilder.setServerApproveBillingEvent(
-          Key.create(
-              serverApproveEntities
-                  .stream()
-                  .filter(BillingEvent.OneTime.class::isInstance)
-                  .map(BillingEvent.OneTime.class::cast)
-                  .collect(onlyElement())));
+          serverApproveEntities.stream()
+              .filter(BillingEvent.OneTime.class::isInstance)
+              .map(BillingEvent.OneTime.class::cast)
+              .collect(onlyElement())
+              .createVKey());
     }
     return transferDataBuilder
         .setTransferStatus(TransferStatus.PENDING)
         .setServerApproveAutorenewEvent(
-            Key.create(
-                serverApproveEntities
-                    .stream()
-                    .filter(BillingEvent.Recurring.class::isInstance)
-                    .map(BillingEvent.Recurring.class::cast)
-                    .collect(onlyElement())))
+            serverApproveEntities.stream()
+                .filter(BillingEvent.Recurring.class::isInstance)
+                .map(BillingEvent.Recurring.class::cast)
+                .collect(onlyElement())
+                .createVKey())
         .setServerApproveAutorenewPollMessage(
-            Key.create(
-                serverApproveEntities
-                    .stream()
-                    .filter(PollMessage.Autorenew.class::isInstance)
-                    .map(PollMessage.Autorenew.class::cast)
-                    .collect(onlyElement())))
+            serverApproveEntities.stream()
+                .filter(PollMessage.Autorenew.class::isInstance)
+                .map(PollMessage.Autorenew.class::cast)
+                .collect(onlyElement())
+                .createVKey())
         .setServerApproveEntities(serverApproveEntityKeys.build())
         .setTransferPeriod(transferPeriod)
         .build();

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -60,7 +60,6 @@ import google.registry.flows.domain.DomainFlowUtils.MissingRegistrantException;
 import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
-import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainCommand.Update;
 import google.registry.model.domain.DomainCommand.Update.AddRemove;
@@ -73,11 +72,9 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
-import google.registry.model.host.HostResource;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.persistence.VKey;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
@@ -247,22 +244,11 @@ public final class DomainUpdateFlow implements TransactionalFlow {
             .setLastEppUpdateClientId(clientId)
             .addStatusValues(add.getStatusValues())
             .removeStatusValues(remove.getStatusValues())
-            .addNameservers(
-                add.getNameservers().stream()
-                    .map(key -> VKey.createOfy(HostResource.class, key))
-                    .collect(toImmutableSet()))
-            .removeNameservers(
-                remove.getNameservers().stream()
-                    .map(key -> VKey.createOfy(HostResource.class, key))
-                    .collect(toImmutableSet()))
+            .addNameservers(add.getNameservers().stream().collect(toImmutableSet()))
+            .removeNameservers(remove.getNameservers().stream().collect(toImmutableSet()))
             .addContacts(add.getContacts())
             .removeContacts(remove.getContacts())
-            .setRegistrant(
-                firstNonNull(
-                    change.getRegistrant() != null
-                        ? VKey.createOfy(ContactResource.class, change.getRegistrant())
-                        : null,
-                    domain.getRegistrant()))
+            .setRegistrant(firstNonNull(change.getRegistrant(), domain.getRegistrant()))
             .setAuthInfo(firstNonNull(change.getAuthInfo(), domain.getAuthInfo()));
     return domainBuilder.build();
   }
@@ -275,7 +261,7 @@ public final class DomainUpdateFlow implements TransactionalFlow {
 
   private void validateNewState(DomainBase newDomain) throws EppException {
     validateNoDuplicateContacts(newDomain.getContacts());
-    validateRequiredContactsPresent(newDomain.getRegistrant().getOfyKey(), newDomain.getContacts());
+    validateRequiredContactsPresent(newDomain.getRegistrant(), newDomain.getContacts());
     validateDsData(newDomain.getDsData());
     validateNameserversCountForTld(
         newDomain.getTld(),

core/src/main/java/google/registry/flows/domain/FeesAndCredits.java

@@ -15,6 +15,7 @@
 package google.registry.flows.domain;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
@@ -27,6 +28,7 @@ import google.registry.model.domain.fee.BaseFee;
 import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Credit;
 import google.registry.model.domain.fee.Fee;
+import java.math.BigDecimal;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 
@@ -39,26 +41,30 @@ public class FeesAndCredits extends ImmutableObject implements Buildable {
   private ImmutableList<Credit> credits;
 
   private Money getTotalCostForType(FeeType type) {
-    Money result = Money.zero(currency);
     checkArgumentNotNull(type);
-    for (Fee fee : fees) {
-      if (fee.getType() == type) {
-        result = result.plus(fee.getCost());
-      }
-    }
-    return result;
+    return Money.of(
+        currency,
+        fees.stream()
+            .filter(f -> f.getType() == type)
+            .map(BaseFee::getCost)
+            .reduce(zeroInCurrency(currency), BigDecimal::add));
+  }
+
+  public boolean hasPremiumFeesOfType(FeeType type) {
+    return fees.stream().filter(f -> f.getType() == type).anyMatch(BaseFee::isPremium);
   }
 
   /** Returns the total cost of all fees and credits for the event. */
   public Money getTotalCost() {
-    Money result = Money.zero(currency);
-    for (Fee fee : fees) {
-      result = result.plus(fee.getCost());
-    }
-    for (Credit credit : credits) {
-      result = result.plus(credit.getCost());
-    }
-    return result;
+    return Money.of(
+        currency,
+        Streams.concat(fees.stream(), credits.stream())
+            .map(BaseFee::getCost)
+            .reduce(zeroInCurrency(currency), BigDecimal::add));
+  }
+
+  public boolean hasAnyPremiumFees() {
+    return fees.stream().anyMatch(BaseFee::isPremium);
   }
 
   /** Returns the create cost for the event. */

core/src/main/java/google/registry/flows/host/HostCreateFlow.java

@@ -128,7 +128,7 @@ public final class HostCreateFlow implements TransactionalFlow {
             .setFullyQualifiedHostName(targetId)
             .setInetAddresses(command.getInetAddresses())
             .setRepoId(createRepoId(ObjectifyService.allocateId(), roidSuffix))
-            .setSuperordinateDomain(superordinateDomain.map(Key::create).orElse(null))
+            .setSuperordinateDomain(superordinateDomain.map(DomainBase::createVKey).orElse(null))
             .build();
     historyBuilder
         .setType(HistoryEntry.Type.HOST_CREATE)

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -96,8 +96,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
       // the client id, needs to be read off of it.
       EppResource owningResource =
           existingHost.isSubordinate()
-              ? ofy().load().key(existingHost.getSuperordinateDomain()).now()
-                  .cloneProjectedAtTime(now)
+              ? tm().load(existingHost.getSuperordinateDomain()).cloneProjectedAtTime(now)
               : existingHost;
       verifyResourceOwnership(clientId, owningResource);
     }

core/src/main/java/google/registry/flows/host/HostInfoFlow.java

@@ -18,7 +18,7 @@ import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.ResourceFlowUtils.loadAndVerifyExistence;
 import static google.registry.flows.host.HostFlowUtils.validateHostName;
 import static google.registry.model.EppResourceUtils.isLinked;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
@@ -77,7 +77,7 @@ public final class HostInfoFlow implements Flow {
     // there is no superordinate domain, the host's own values for these fields will be correct.
     if (host.isSubordinate()) {
       DomainBase superordinateDomain =
-          ofy().load().key(host.getSuperordinateDomain()).now().cloneProjectedAtTime(now);
+          tm().load(host.getSuperordinateDomain()).cloneProjectedAtTime(now);
       hostInfoDataBuilder
           .setCurrentSponsorClientId(superordinateDomain.getCurrentSponsorClientId())
           .setLastTransferTime(host.computeLastTransferTime(superordinateDomain));

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -60,6 +60,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
+import google.registry.persistence.VKey;
 import java.util.Objects;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -136,9 +137,10 @@ public final class HostUpdateFlow implements TransactionalFlow {
     boolean isHostRename = suppliedNewHostName != null;
     String oldHostName = targetId;
     String newHostName = firstNonNull(suppliedNewHostName, oldHostName);
-    DomainBase oldSuperordinateDomain = existingHost.isSubordinate()
-        ? ofy().load().key(existingHost.getSuperordinateDomain()).now().cloneProjectedAtTime(now)
-        : null;
+    DomainBase oldSuperordinateDomain =
+        existingHost.isSubordinate()
+            ? tm().load(existingHost.getSuperordinateDomain()).cloneProjectedAtTime(now)
+            : null;
     // Note that lookupSuperordinateDomain calls cloneProjectedAtTime on the domain for us.
     Optional<DomainBase> newSuperordinateDomain =
         lookupSuperordinateDomain(validateHostName(newHostName), now);
@@ -153,8 +155,8 @@ public final class HostUpdateFlow implements TransactionalFlow {
     AddRemove remove = command.getInnerRemove();
     checkSameValuesNotAddedAndRemoved(add.getStatusValues(), remove.getStatusValues());
     checkSameValuesNotAddedAndRemoved(add.getInetAddresses(), remove.getInetAddresses());
-    Key<DomainBase> newSuperordinateDomainKey =
-        newSuperordinateDomain.map(Key::create).orElse(null);
+    VKey<DomainBase> newSuperordinateDomainKey =
+        newSuperordinateDomain.map(DomainBase::createVKey).orElse(null);
     // If the superordinateDomain field is changing, set the lastSuperordinateChange to now.
     DateTime lastSuperordinateChange =
         Objects.equals(newSuperordinateDomainKey, existingHost.getSuperordinateDomain())
@@ -280,28 +282,28 @@ public final class HostUpdateFlow implements TransactionalFlow {
     if (existingHost.isSubordinate()
         && newHost.isSubordinate()
         && Objects.equals(
-            existingHost.getSuperordinateDomain(),
-            newHost.getSuperordinateDomain())) {
-      ofy().save().entity(
-          ofy().load().key(existingHost.getSuperordinateDomain()).now().asBuilder()
-              .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
-              .addSubordinateHost(newHost.getFullyQualifiedHostName())
-              .build());
+            existingHost.getSuperordinateDomain(), newHost.getSuperordinateDomain())) {
+      tm().saveNewOrUpdate(
+              tm().load(existingHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
+                  .addSubordinateHost(newHost.getFullyQualifiedHostName())
+                  .build());
       return;
     }
     if (existingHost.isSubordinate()) {
-      ofy().save().entity(
-          ofy().load().key(existingHost.getSuperordinateDomain()).now()
-              .asBuilder()
-              .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
-              .build());
+      tm().saveNewOrUpdate(
+              tm().load(existingHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
+                  .build());
     }
     if (newHost.isSubordinate()) {
-      ofy().save().entity(
-          ofy().load().key(newHost.getSuperordinateDomain()).now()
-              .asBuilder()
-              .addSubordinateHost(newHost.getFullyQualifiedHostName())
-              .build());
+      tm().saveNewOrUpdate(
+              tm().load(newHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .addSubordinateHost(newHost.getFullyQualifiedHostName())
+                  .build());
     }
   }
 

core/src/main/java/google/registry/model/EppResource.java

@@ -16,12 +16,11 @@ package google.registry.model;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
@@ -32,11 +31,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Index;
@@ -44,12 +41,13 @@ import google.registry.config.RegistryConfig;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.transfer.TransferData;
+import google.registry.persistence.VKey;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.StreamSupport;
 import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
@@ -70,11 +68,11 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /** The ID of the registrar that is currently sponsoring this resource. */
   @Index
-  @Column(nullable = false)
+  @Column(name = "currentSponsorRegistrarId", nullable = false)
   String currentSponsorClientId;
 
   /** The ID of the registrar that created this resource. */
-  @Column(nullable = false)
+  @Column(name = "creationRegistrarId", nullable = false)
   String creationClientId;
 
   /**
@@ -84,6 +82,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * edits; it only includes EPP-visible modifications such as {@literal <update>}. Can be null if
    * the resource has never been modified.
    */
+  @Column(name = "lastEppUpdateRegistrarId")
   String lastEppUpdateClientId;
 
   /** The time when this resource was created. */
@@ -183,6 +182,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   /** Get the foreign key string for this resource. */
   public abstract String getForeignKey();
 
+  /** Create the VKey for the specified EPP resource. */
+  public abstract VKey<? extends EppResource> createVKey();
+
   /** Override of {@link Buildable#asBuilder} so that the extra methods are visible. */
   @Override
   public abstract Builder<?, ?> asBuilder();
@@ -329,18 +331,18 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     }
   }
 
-  static final CacheLoader<Key<? extends EppResource>, EppResource> CACHE_LOADER =
-      new CacheLoader<Key<? extends EppResource>, EppResource>() {
+  static final CacheLoader<VKey<? extends EppResource>, EppResource> CACHE_LOADER =
+      new CacheLoader<VKey<? extends EppResource>, EppResource>() {
 
         @Override
-        public EppResource load(Key<? extends EppResource> key) {
-          return tm().doTransactionless(() -> ofy().load().key(key).now());
+        public EppResource load(VKey<? extends EppResource> key) {
+          return tm().doTransactionless(() -> tm().load(key));
         }
 
         @Override
-        public Map<Key<? extends EppResource>, EppResource> loadAll(
-            Iterable<? extends Key<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> loadMultiple(keys));
+        public Map<VKey<? extends EppResource>, EppResource> loadAll(
+            Iterable<? extends VKey<? extends EppResource>> keys) {
+          return tm().doTransactionless(() -> loadAsMap(keys));
         }
       };
 
@@ -352,10 +354,10 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * directly should of course never use the cache.
    */
   @NonFinalForTesting
-  private static LoadingCache<Key<? extends EppResource>, EppResource> cacheEppResources =
+  private static LoadingCache<VKey<? extends EppResource>, EppResource> cacheEppResources =
       createEppResourcesCache(getEppResourceCachingDuration());
 
-  private static LoadingCache<Key<? extends EppResource>, EppResource> createEppResourcesCache(
+  private static LoadingCache<VKey<? extends EppResource>, EppResource> createEppResourcesCache(
       Duration expiry) {
     return CacheBuilder.newBuilder()
         .expireAfterWrite(expiry.getMillis(), MILLISECONDS)
@@ -369,29 +371,16 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     cacheEppResources = createEppResourcesCache(effectiveExpiry);
   }
 
-  private static ImmutableMap<Key<? extends EppResource>, EppResource> loadMultiple(
-      Iterable<? extends Key<? extends EppResource>> keys) {
-    // This cast is safe because, in Objectify, Key<? extends EppResource> can also be
-    // treated as a Key<EppResource>.
-    @SuppressWarnings("unchecked")
-    ImmutableList<Key<EppResource>> typedKeys =
-        Streams.stream(keys).map(key -> (Key<EppResource>) key).collect(toImmutableList());
-
-    // Typing shenanigans are required to return the map with the correct required generic type.
-    return ofy().load().keys(typedKeys).entrySet().stream()
-        .collect(ImmutableMap.toImmutableMap(Entry::getKey, Entry::getValue));
-  }
-
   /**
    * Loads the given EppResources by their keys using the cache (if enabled).
    *
    * <p>Don't use this unless you really need it for performance reasons, and be sure that you are
    * OK with the trade-offs in loss of transactional consistency.
    */
-  public static ImmutableMap<Key<? extends EppResource>, EppResource> loadCached(
-      Iterable<Key<? extends EppResource>> keys) {
+  public static ImmutableMap<VKey<? extends EppResource>, EppResource> loadCached(
+      Iterable<VKey<? extends EppResource>> keys) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return loadMultiple(keys);
+      return loadAsMap(keys);
     }
     try {
       return cacheEppResources.getAll(keys);
@@ -406,9 +395,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>Don't use this unless you really need it for performance reasons, and be sure that you are
    * OK with the trade-offs in loss of transactional consistency.
    */
-  public static <T extends EppResource> T loadCached(Key<T> key) {
+  public static <T extends EppResource> T loadCached(VKey<T> key) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return ofy().load().key(key).now();
+      return tm().load(key);
     }
     try {
       // Safe to cast because loading a Key<T> returns an entity of type T.
@@ -419,4 +408,17 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
       throw new RuntimeException("Error loading cached EppResources", e.getCause());
     }
   }
+
+  private static ImmutableMap<VKey<? extends EppResource>, EppResource> loadAsMap(
+      Iterable<? extends VKey<? extends EppResource>> keys) {
+    return StreamSupport.stream(keys.spliterator(), false)
+        // It's possible for us to receive the same key more than once which causes
+        // the immutable map build to break with a duplicate key, so we have to ensure key
+        // uniqueness.
+        .distinct()
+        // We have to use "key -> key" here instead of the identity() function, because
+        // the latter breaks the fairly complicated generic type checking required by the
+        // caching interface.
+        .collect(toImmutableMap(key -> key, key -> tm().load(key)));
+  }
 }

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -17,6 +17,7 @@ package google.registry.model;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;
@@ -141,7 +142,7 @@ public final class EppResourceUtils {
     T resource =
         useCache
             ? EppResource.loadCached(fki.getResourceKey())
-            : ofy().load().key(fki.getResourceKey()).now();
+            : tm().maybeLoad(fki.getResourceKey()).orElse(null);
     if (resource == null || isAtOrAfter(now, resource.getDeletionTime())) {
       return Optional.empty();
     }

core/src/main/java/google/registry/model/ResourceTransferUtils.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -114,7 +115,7 @@ public final class ResourceTransferUtils {
             R resource, R newResource, DateTime now, HistoryEntry historyEntry) {
     if (resource.getStatusValues().contains(StatusValue.PENDING_TRANSFER)) {
       TransferData oldTransferData = resource.getTransferData();
-      ofy().delete().keys(oldTransferData.getServerApproveEntities());
+      tm().delete(oldTransferData.getServerApproveEntities());
       ofy()
           .save()
           .entity(

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -116,7 +116,7 @@ public abstract class BillingEvent extends ImmutableObject
 
   /** The registrar to bill. */
   @Index
-  @Column(nullable = false)
+  @Column(name = "registrarId", nullable = false)
   String clientId;
 
   /** Revision id of the entry in DomainHistory table that ths bill belongs to. */
@@ -267,7 +267,7 @@ public abstract class BillingEvent extends ImmutableObject
   @javax.persistence.Entity(name = "BillingEvent")
   @javax.persistence.Table(
       indexes = {
-        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "registrarId"),
         @javax.persistence.Index(columnList = "eventTime"),
         @javax.persistence.Index(columnList = "billingTime"),
         @javax.persistence.Index(columnList = "syntheticCreationTime"),
@@ -345,6 +345,7 @@ public abstract class BillingEvent extends ImmutableObject
       return Optional.ofNullable(allocationToken);
     }
 
+    @Override
     public VKey<OneTime> createVKey() {
       return VKey.createOfy(getClass(), Key.create(this));
     }
@@ -439,7 +440,7 @@ public abstract class BillingEvent extends ImmutableObject
   @javax.persistence.Entity(name = "BillingRecurrence")
   @javax.persistence.Table(
       indexes = {
-        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "registrarId"),
         @javax.persistence.Index(columnList = "eventTime"),
         @javax.persistence.Index(columnList = "recurrenceEndTime"),
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
@@ -482,6 +483,7 @@ public abstract class BillingEvent extends ImmutableObject
       return recurrenceTimeOfYear;
     }
 
+    @Override
     public VKey<Recurring> createVKey() {
       return VKey.createOfy(getClass(), Key.create(this));
     }
@@ -529,7 +531,7 @@ public abstract class BillingEvent extends ImmutableObject
   @javax.persistence.Entity(name = "BillingCancellation")
   @javax.persistence.Table(
       indexes = {
-        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "registrarId"),
         @javax.persistence.Index(columnList = "eventTime"),
         @javax.persistence.Index(columnList = "billingTime")
       })
@@ -603,6 +605,7 @@ public abstract class BillingEvent extends ImmutableObject
       return builder.build();
     }
 
+    @Override
     public VKey<Cancellation> createVKey() {
       return VKey.createOfy(getClass(), Key.create(this));
     }
@@ -682,6 +685,11 @@ public abstract class BillingEvent extends ImmutableObject
       return new Builder(clone(this));
     }
 
+    @Override
+    public VKey<Modification> createVKey() {
+      return VKey.createOfy(this.getClass(), Key.create(this));
+    }
+
     /**
      * Create a new Modification billing event which is a refund of the given OneTime billing event
      * and that is parented off the given HistoryEntry.

core/src/main/java/google/registry/model/common/Cursor.java

@@ -35,9 +35,9 @@ import java.util.List;
 import org.joda.time.DateTime;
 
 /**
- * Shared entity for date cursors. This type supports both "scoped" cursors (i.e. per resource
- * of a given type, such as a TLD) and global (i.e. one per environment) cursors, defined internally
- * as scoped on {@link EntityGroupRoot}.
+ * Shared entity for date cursors. This type supports both "scoped" cursors (i.e. per resource of a
+ * given type, such as a TLD) and global (i.e. one per environment) cursors, defined internally as
+ * scoped on {@link EntityGroupRoot}.
  */
 @Entity
 public class Cursor extends ImmutableObject implements DatastoreEntity {

core/src/main/java/google/registry/model/common/EntityGroupRoot.java

@@ -27,14 +27,14 @@ import google.registry.schema.replay.SqlEntity;
  * reasons.
  *
  * <p>This exists as a storage place for common configuration options and global settings that
- * aren't updated too frequently.  Entities in this entity group are usually cached upon load.  The
+ * aren't updated too frequently. Entities in this entity group are usually cached upon load. The
  * reason this common entity group exists is because it enables strongly consistent queries and
- * updates across this seldomly updated data.  This shared entity group also helps cut down on
- * a potential ballooning in the number of entity groups enlisted in transactions.
+ * updates across this seldomly updated data. This shared entity group also helps cut down on a
+ * potential ballooning in the number of entity groups enlisted in transactions.
  *
- * <p>Historically, each TLD used to have a separate namespace, and all entities for a TLD were in
- * a single EntityGroupRoot for that TLD.  Hence why there was a "cross-tld" entity group -- it was
- * the entity group for the single namespace where global data applicable for all TLDs lived.
+ * <p>Historically, each TLD used to have a separate namespace, and all entities for a TLD were in a
+ * single EntityGroupRoot for that TLD. Hence why there was a "cross-tld" entity group -- it was the
+ * entity group for the single namespace where global data applicable for all TLDs lived.
  */
 @Entity
 public class EntityGroupRoot extends BackupGroupRoot implements DatastoreEntity {

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -41,7 +41,6 @@ import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Embedded;
-import javax.persistence.Transient;
 import javax.xml.bind.annotation.XmlElement;
 import org.joda.time.DateTime;
 
@@ -57,7 +56,7 @@ import org.joda.time.DateTime;
     name = "Contact",
     indexes = {
       @javax.persistence.Index(columnList = "creationTime"),
-      @javax.persistence.Index(columnList = "currentSponsorClientId"),
+      @javax.persistence.Index(columnList = "currentSponsorRegistrarId"),
       @javax.persistence.Index(columnList = "deletionTime"),
       @javax.persistence.Index(columnList = "contactId", unique = true),
       @javax.persistence.Index(columnList = "searchName")
@@ -171,8 +170,7 @@ public class ContactResource extends EppResource
   ContactAuthInfo authInfo;
 
   /** Data about any pending or past transfers on this contact. */
-  // TODO(b/153363295): Figure out how to persist transfer data
-  @Transient TransferData transferData;
+  TransferData transferData;
 
   /**
    * The time that this resource was last transferred.
@@ -197,7 +195,9 @@ public class ContactResource extends EppResource
   })
   Disclose disclose;
 
+  @Override
   public VKey<ContactResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.createOfy(ContactResource.class, Key.create(this));
   }
 

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -66,6 +66,7 @@ import google.registry.model.registry.Registry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.util.CollectionUtils;
 import java.util.HashSet;
@@ -101,11 +102,12 @@ import org.joda.time.Interval;
     name = "Domain",
     indexes = {
       @javax.persistence.Index(columnList = "creationTime"),
-      @javax.persistence.Index(columnList = "currentSponsorClientId"),
+      @javax.persistence.Index(columnList = "currentSponsorRegistrarId"),
       @javax.persistence.Index(columnList = "deletionTime"),
       @javax.persistence.Index(columnList = "fullyQualifiedDomainName"),
       @javax.persistence.Index(columnList = "tld")
     })
+@WithStringVKey
 @ExternalMessagingName("domain")
 public class DomainBase extends EppResource
     implements DatastoreAndSqlEntity, ForeignKeyedEppResource, ResourceWithTransferData {
@@ -251,7 +253,7 @@ public class DomainBase extends EppResource
   String smdId;
 
   /** Data about any pending or past transfers on this domain. */
-  @Transient TransferData transferData;
+  TransferData transferData;
 
   /**
    * The time that this resource was last transferred.
@@ -436,8 +438,14 @@ public class DomainBase extends EppResource
               .setRegistrationExpirationTime(expirationDate)
               // Set the speculatively-written new autorenew events as the domain's autorenew
               // events.
-              .setAutorenewBillingEvent(transferData.getServerApproveAutorenewEvent())
-              .setAutorenewPollMessage(transferData.getServerApproveAutorenewPollMessage());
+              .setAutorenewBillingEvent(
+                  transferData.getServerApproveAutorenewEvent() == null
+                      ? null
+                      : transferData.getServerApproveAutorenewEvent().getOfyKey())
+              .setAutorenewPollMessage(
+                  transferData.getServerApproveAutorenewPollMessage() == null
+                      ? null
+                      : transferData.getServerApproveAutorenewPollMessage().getOfyKey());
       if (transferData.getTransferPeriod().getValue() == 1) {
         // Set the grace period using a key to the prescheduled transfer billing event.  Not using
         // GracePeriod.forBillingEvent() here in order to avoid the actual Datastore fetch.
@@ -448,7 +456,9 @@ public class DomainBase extends EppResource
                     transferExpirationTime.plus(
                         Registry.get(getTld()).getTransferGracePeriodLength()),
                     transferData.getGainingClientId(),
-                    transferData.getServerApproveBillingEvent())));
+                    transferData.getServerApproveBillingEvent() == null
+                        ? null
+                        : transferData.getServerApproveBillingEvent().getOfyKey())));
       } else {
         // There won't be a billing event, so we don't need a grace period
         builder.setGracePeriods(ImmutableSet.of());
@@ -613,6 +623,12 @@ public class DomainBase extends EppResource
     }
   }
 
+  @Override
+  public VKey<DomainBase> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.createOfy(DomainBase.class, Key.create(this));
+  }
+
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
   private static final Predicate<DesignatedContact> IS_REGISTRANT =
       (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);

core/src/main/java/google/registry/model/domain/DomainCommand.java

@@ -30,7 +30,6 @@ import com.google.common.base.MoreObjects;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.contact.ContactResource;
@@ -82,8 +81,7 @@ public class DomainCommand {
     String registrantContactId;
 
     /** A resolved key to the registrant who registered this domain. */
-    @XmlTransient
-    Key<ContactResource> registrant;
+    @XmlTransient VKey<ContactResource> registrant;
 
     /** Authorization info (aka transfer secret) of the domain. */
     DomainAuthInfo authInfo;
@@ -93,7 +91,7 @@ public class DomainCommand {
     }
 
     @Nullable
-    public Key<ContactResource> getRegistrant() {
+    public VKey<ContactResource> getRegistrant() {
       return registrant;
     }
 
@@ -129,8 +127,7 @@ public class DomainCommand {
     Set<String> nameserverFullyQualifiedHostNames;
 
     /** Resolved keys to hosts that are the nameservers for the domain. */
-    @XmlTransient
-    Set<Key<HostResource>> nameservers;
+    @XmlTransient Set<VKey<HostResource>> nameservers;
 
     /** Foreign keyed associated contacts for the domain (other than registrant). */
     @XmlElement(name = "contact")
@@ -160,7 +157,7 @@ public class DomainCommand {
       return nullToEmptyImmutableCopy(nameserverFullyQualifiedHostNames);
     }
 
-    public ImmutableSet<Key<HostResource>> getNameservers() {
+    public ImmutableSet<VKey<HostResource>> getNameservers() {
       return nullToEmptyImmutableCopy(nameservers);
     }
 
@@ -190,7 +187,7 @@ public class DomainCommand {
             now);
         for (DesignatedContact contact : contacts) {
           if (DesignatedContact.Type.REGISTRANT.equals(contact.getType())) {
-            clone.registrant = contact.getContactKey().getOfyKey();
+            clone.registrant = contact.getContactKey();
             clone.contacts = forceEmptyToNull(difference(contacts, contact));
             break;
           }
@@ -354,8 +351,7 @@ public class DomainCommand {
       Set<String> nameserverFullyQualifiedHostNames;
 
       /** Resolved keys to hosts that are the nameservers for the domain. */
-      @XmlTransient
-      Set<Key<HostResource>> nameservers;
+      @XmlTransient Set<VKey<HostResource>> nameservers;
 
       /** Foreign keyed associated contacts for the domain (other than registrant). */
       @XmlElement(name = "contact")
@@ -369,7 +365,7 @@ public class DomainCommand {
         return nullSafeImmutableCopy(nameserverFullyQualifiedHostNames);
       }
 
-      public ImmutableSet<Key<HostResource>> getNameservers() {
+      public ImmutableSet<VKey<HostResource>> getNameservers() {
         return nullToEmptyImmutableCopy(nameservers);
       }
 
@@ -419,7 +415,7 @@ public class DomainCommand {
     }
   }
 
-  private static Set<Key<HostResource>> linkHosts(
+  private static Set<VKey<HostResource>> linkHosts(
       Set<String> fullyQualifiedHostNames, DateTime now) throws InvalidReferencesException {
     if (fullyQualifiedHostNames == null) {
       return null;
@@ -437,20 +433,18 @@ public class DomainCommand {
     for (ForeignKeyedDesignatedContact contact : contacts) {
       foreignKeys.add(contact.contactId);
     }
-    ImmutableMap<String, Key<ContactResource>> loadedContacts =
+    ImmutableMap<String, VKey<ContactResource>> loadedContacts =
         loadByForeignKeysCached(foreignKeys.build(), ContactResource.class, now);
     ImmutableSet.Builder<DesignatedContact> linkedContacts = new ImmutableSet.Builder<>();
     for (ForeignKeyedDesignatedContact contact : contacts) {
       linkedContacts.add(
-          DesignatedContact.create(
-              contact.type,
-              VKey.createOfy(ContactResource.class, loadedContacts.get(contact.contactId))));
+          DesignatedContact.create(contact.type, loadedContacts.get(contact.contactId)));
     }
     return linkedContacts.build();
   }
 
   /** Loads keys to cached EPP resources by their foreign keys. */
-  private static <T extends EppResource> ImmutableMap<String, Key<T>> loadByForeignKeysCached(
+  private static <T extends EppResource> ImmutableMap<String, VKey<T>> loadByForeignKeysCached(
       final Set<String> foreignKeys, final Class<T> clazz, final DateTime now)
       throws InvalidReferencesException {
     Map<String, ForeignKeyIndex<T>> fkis = ForeignKeyIndex.loadCached(clazz, foreignKeys, now);

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -24,6 +24,7 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import org.joda.time.DateTime;
@@ -51,6 +52,7 @@ public class GracePeriod extends ImmutableObject {
   DateTime expirationTime;
 
   /** The registrar to bill. */
+  @Column(name = "registrarId")
   String clientId;
 
   /**

core/src/main/java/google/registry/model/domain/Period.java

@@ -16,6 +16,8 @@ package google.registry.model.domain;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlEnumValue;
 import javax.xml.bind.annotation.XmlValue;
@@ -25,6 +27,7 @@ import javax.xml.bind.annotation.XmlValue;
 @javax.persistence.Embeddable
 public class Period extends ImmutableObject {
 
+  @Enumerated(EnumType.STRING)
   @XmlAttribute
   Unit unit;
 

core/src/main/java/google/registry/model/domain/fee/BaseFee.java

@@ -104,6 +104,8 @@ public abstract class BaseFee extends ImmutableObject {
 
   @XmlTransient Range<DateTime> validDateRange;
 
+  @XmlTransient boolean isPremium;
+
   public String getDescription() {
     return description;
   }
@@ -120,6 +122,11 @@ public abstract class BaseFee extends ImmutableObject {
     return firstNonNull(refundable, true);
   }
 
+  /** Returns whether the fee in question is a premium price. */
+  public boolean isPremium() {
+    return isPremium;
+  }
+
   /**
    * According to the fee extension specification, a fee must always be non-negative, while a credit
    * must always be negative. Essentially, they are the same thing, just with different sign.

core/src/main/java/google/registry/model/domain/fee/Fee.java

@@ -31,25 +31,33 @@ import org.joda.time.DateTime;
 public class Fee extends BaseFee {
 
   /** Creates a Fee for the given cost and type with the default description. */
-  public static Fee create(BigDecimal cost, FeeType type, Object... descriptionArgs) {
+  public static Fee create(
+      BigDecimal cost, FeeType type, boolean isPremium, Object... descriptionArgs) {
     checkArgumentNotNull(type, "Must specify the type of the fee");
-    return createWithCustomDescription(cost, type, type.renderDescription(descriptionArgs));
+    return createWithCustomDescription(
+        cost, type, isPremium, type.renderDescription(descriptionArgs));
   }
 
   /** Creates a Fee for the given cost, type, and valid date range with the default description. */
   public static Fee create(
-      BigDecimal cost, FeeType type, Range<DateTime> validDateRange, Object... descriptionArgs) {
-    Fee instance = create(cost, type, descriptionArgs);
+      BigDecimal cost,
+      FeeType type,
+      boolean isPremium,
+      Range<DateTime> validDateRange,
+      Object... descriptionArgs) {
+    Fee instance = create(cost, type, isPremium, descriptionArgs);
     instance.validDateRange = validDateRange;
     return instance;
   }
 
   /** Creates a Fee for the given cost and type with a custom description. */
-  public static Fee createWithCustomDescription(BigDecimal cost, FeeType type, String description) {
+  private static Fee createWithCustomDescription(
+      BigDecimal cost, FeeType type, boolean isPremium, String description) {
     Fee instance = new Fee();
     instance.cost = checkNotNull(cost);
-    checkArgument(instance.cost.signum() >= 0);
+    checkArgument(instance.cost.signum() >= 0, "Cost must be a positive number");
     instance.type = checkNotNull(type);
+    instance.isPremium = isPremium;
     instance.description = description;
     return instance;
   }

core/src/main/java/google/registry/model/domain/fee11/FeeCheckCommandExtensionV11.java

@@ -56,7 +56,7 @@ public class FeeCheckCommandExtensionV11 extends ImmutableObject
   /** The period to check. */
   Period period;
 
-  /** The class to check. */
+  /** The fee class to check. */
   @XmlElement(name = "class")
   String feeClass;
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -40,7 +40,6 @@ import java.net.InetAddress;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
-import javax.persistence.ElementCollection;
 import org.joda.time.DateTime;
 
 /**
@@ -70,13 +69,13 @@ public class HostResource extends EppResource
   String fullyQualifiedHostName;
 
   /** IP Addresses for this host. Can be null if this is an external host. */
-  @Index @ElementCollection Set<InetAddress> inetAddresses;
+  @Index Set<InetAddress> inetAddresses;
 
   /** The superordinate domain of this host, or null if this is an external host. */
   @Index
   @IgnoreSave(IfNull.class)
   @DoNotHydrate
-  Key<DomainBase> superordinateDomain;
+  VKey<DomainBase> superordinateDomain;
 
   /**
    * The time that this resource was last transferred.
@@ -98,7 +97,7 @@ public class HostResource extends EppResource
     return fullyQualifiedHostName;
   }
 
-  public Key<DomainBase> getSuperordinateDomain() {
+  public VKey<DomainBase> getSuperordinateDomain() {
     return superordinateDomain;
   }
 
@@ -123,7 +122,9 @@ public class HostResource extends EppResource
     return fullyQualifiedHostName;
   }
 
-  public VKey<HostResource> createKey() {
+  @Override
+  public VKey<HostResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.createOfy(HostResource.class, Key.create(this));
   }
 
@@ -143,18 +144,18 @@ public class HostResource extends EppResource
    *
    * <p>If the host is not subordinate the domain can be null and we just return last transfer time.
    *
-   * @param superordinateDomain the loaded superordinate domain, which must match the key in
-   *     the {@link #superordinateDomain} field. Passing it as a parameter allows the caller to
-   *     control the degree of consistency used to load it.
+   * @param superordinateDomain the loaded superordinate domain, which must match the key in the
+   *     {@link #superordinateDomain} field. Passing it as a parameter allows the caller to control
+   *     the degree of consistency used to load it.
    */
-   public DateTime computeLastTransferTime(@Nullable DomainBase superordinateDomain) {
+  public DateTime computeLastTransferTime(@Nullable DomainBase superordinateDomain) {
     if (!isSubordinate()) {
       checkArgument(superordinateDomain == null);
       return getLastTransferTime();
     }
     checkArgument(
         superordinateDomain != null
-            && Key.create(superordinateDomain).equals(getSuperordinateDomain()));
+            && superordinateDomain.createVKey().equals(getSuperordinateDomain()));
     DateTime lastSuperordinateChange =
         Optional.ofNullable(getLastSuperordinateChange()).orElse(getCreationTime());
     DateTime lastTransferOfCurrentSuperordinate =
@@ -205,7 +206,7 @@ public class HostResource extends EppResource
           difference(getInstance().getInetAddresses(), inetAddresses)));
     }
 
-    public Builder setSuperordinateDomain(Key<DomainBase> superordinateDomain) {
+    public Builder setSuperordinateDomain(VKey<DomainBase> superordinateDomain) {
       getInstance().superordinateDomain = superordinateDomain;
       return this;
     }

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -43,6 +43,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
 import java.util.Optional;
@@ -99,7 +100,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * <p>This field holds a key to the only referenced resource. It is named "topReference" for
    * historical reasons.
    */
-  Key<E> topReference;
+  VKey<E> topReference;
 
   public String getForeignKey() {
     return foreignKey;
@@ -109,7 +110,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     return deletionTime;
   }
 
-  public Key<E> getResourceKey() {
+  public VKey<E> getResourceKey() {
     return topReference;
   }
 
@@ -125,7 +126,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     @SuppressWarnings("unchecked")
     Class<E> resourceClass = (Class<E>) resource.getClass();
     ForeignKeyIndex<E> instance = instantiate(mapToFkiClass(resourceClass));
-    instance.topReference = Key.create(resource);
+    instance.topReference = (VKey<E>) resource.createVKey();
     instance.foreignKey = resource.getForeignKey();
     instance.deletionTime = deletionTime;
     return instance;
@@ -141,18 +142,18 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   /**
    * Loads a {@link Key} to an {@link EppResource} from Datastore by foreign key.
    *
-   * <p>Returns null if no foreign key index with this foreign key was ever created, or if the
-   * most recently created foreign key index was deleted before time "now". This method does not
-   * actually check that the referenced resource actually exists. However, for normal epp resources,
-   * it is safe to assume that the referenced resource exists if the foreign key index does.
+   * <p>Returns null if no foreign key index with this foreign key was ever created, or if the most
+   * recently created foreign key index was deleted before time "now". This method does not actually
+   * check that the referenced resource actually exists. However, for normal epp resources, it is
+   * safe to assume that the referenced resource exists if the foreign key index does.
    *
    * @param clazz the resource type to load
    * @param foreignKey id to match
    * @param now the current logical time to use when checking for soft deletion of the foreign key
-   *        index
+   *     index
    */
   @Nullable
-  public static <E extends EppResource> Key<E> loadAndGetKey(
+  public static <E extends EppResource> VKey<E> loadAndGetKey(
       Class<E> clazz, String foreignKey, DateTime now) {
     ForeignKeyIndex<E> index = load(clazz, foreignKey, now);
     return (index == null) ? null : index.getResourceKey();

core/src/main/java/google/registry/model/ofy/CommitLogBucket.java

@@ -44,10 +44,9 @@ import org.joda.time.DateTime;
 /**
  * Root for a random commit log bucket.
  *
- * <p>This is used to shard {@link CommitLogManifest} objects into
- * {@link RegistryConfig#getCommitLogBucketCount() N} entity groups. This increases
- * transaction throughput, while maintaining the ability to perform strongly-consistent ancestor
- * queries.
+ * <p>This is used to shard {@link CommitLogManifest} objects into {@link
+ * RegistryConfig#getCommitLogBucketCount() N} entity groups. This increases transaction throughput,
+ * while maintaining the ability to perform strongly-consistent ancestor queries.
  *
  * @see <a href="https://cloud.google.com/appengine/articles/scaling/contention">Avoiding Datastore
  *     contention</a>

core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java

@@ -38,11 +38,11 @@ import org.joda.time.DateTime;
  * Entity representing a point-in-time consistent view of Datastore, based on commit logs.
  *
  * <p>Conceptually, this entity consists of two pieces of information: the checkpoint "wall" time
- * and a set of bucket checkpoint times.  The former is the ID for this checkpoint (constrained
- * to be unique upon checkpoint creation) and also represents the approximate wall time of the
- * consistent Datastore view this checkpoint represents.  The latter is really a mapping from
- * bucket ID to timestamp, where the timestamp dictates the upper bound (inclusive) on commit logs
- * from that bucket to include when restoring Datastore to this checkpoint.
+ * and a set of bucket checkpoint times. The former is the ID for this checkpoint (constrained to be
+ * unique upon checkpoint creation) and also represents the approximate wall time of the consistent
+ * Datastore view this checkpoint represents. The latter is really a mapping from bucket ID to
+ * timestamp, where the timestamp dictates the upper bound (inclusive) on commit logs from that
+ * bucket to include when restoring Datastore to this checkpoint.
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)

core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java

@@ -28,9 +28,7 @@ import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import org.joda.time.DateTime;
 
-/**
- * Singleton parent entity for all commit log checkpoints.
- */
+/** Singleton parent entity for all commit log checkpoints. */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
 public class CommitLogCheckpointRoot extends ImmutableObject implements DatastoreEntity {

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -14,6 +14,7 @@
 
 package google.registry.model.ofy;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -157,7 +158,7 @@ public class DatastoreTransactionManager implements TransactionManager {
   @Override
   public <T> ImmutableList<T> load(Iterable<VKey<T>> keys) {
     Iterator<Key<T>> iter =
-        StreamSupport.stream(keys.spliterator(), false).map(key -> key.getOfyKey()).iterator();
+        StreamSupport.stream(keys.spliterator(), false).map(VKey::getOfyKey).iterator();
 
     // The lambda argument to keys() effectively converts Iterator -> Iterable.
     return ImmutableList.copyOf(getOfy().load().keys(() -> iter).values());
@@ -170,7 +171,18 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public <T> void delete(VKey<T> key) {
+  public void delete(VKey<?> key) {
     getOfy().delete().key(key.getOfyKey()).now();
   }
+
+  @Override
+  public void delete(Iterable<? extends VKey<?>> vKeys) {
+    // We have to create a list to work around the wildcard capture issue here.
+    // See https://docs.oracle.com/javase/tutorial/java/generics/capture.html
+    ImmutableList<Key<?>> list =
+        StreamSupport.stream(vKeys.spliterator(), false)
+            .map(VKey::getOfyKey)
+            .collect(toImmutableList());
+    getOfy().delete().keys(list).now();
+  }
 }

core/src/main/java/google/registry/model/poll/PendingActionNotificationResponse.java

@@ -19,6 +19,7 @@ import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
+import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
@@ -29,11 +30,12 @@ import org.joda.time.DateTime;
 
 /** The {@link ResponseData} returned when completing a pending action on a domain. */
 @XmlTransient
-public abstract class PendingActionNotificationResponse
-    extends ImmutableObject implements ResponseData {
+@Embeddable
+public class PendingActionNotificationResponse extends ImmutableObject implements ResponseData {
 
   /** The inner name type that contains a name and the result boolean. */
   @Embed
+  @Embeddable
   static class NameOrId extends ImmutableObject {
     @XmlValue
     String value;

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -25,6 +25,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
@@ -39,10 +40,24 @@ import google.registry.model.poll.PendingActionNotificationResponse.DomainPendin
 import google.registry.model.poll.PendingActionNotificationResponse.HostPendingActionNotificationResponse;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
+import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferResponse.ContactTransferResponse;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithLongVKey;
 import java.util.List;
 import java.util.Optional;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.DiscriminatorColumn;
+import javax.persistence.DiscriminatorValue;
+import javax.persistence.Embedded;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Inheritance;
+import javax.persistence.InheritanceType;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -68,28 +83,52 @@ import org.joda.time.DateTime;
 @Entity
 @ReportedOn
 @ExternalMessagingName("message")
+@javax.persistence.Entity
+@Inheritance(strategy = InheritanceType.SINGLE_TABLE)
+@DiscriminatorColumn(name = "type")
+@javax.persistence.Table(
+    indexes = {
+      @javax.persistence.Index(columnList = "registrar_id"),
+      @javax.persistence.Index(columnList = "eventTime")
+    })
 public abstract class PollMessage extends ImmutableObject
     implements Buildable, TransferServerApproveEntity {
 
   /** Entity id. */
   @Id
-  long id;
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  @Column(name = "poll_message_id")
+  Long id;
 
-  @Parent
-  @DoNotHydrate
-  Key<HistoryEntry> parent;
+  @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
 
   /** The registrar that this poll message will be delivered to. */
   @Index
+  @Column(name = "registrar_id", nullable = false)
   String clientId;
 
   /** The time when the poll message should be delivered. May be in the future. */
   @Index
+  @Column(nullable = false)
   DateTime eventTime;
 
   /** Human readable message that will be returned with this poll message. */
+  @Column(name = "message")
   String msg;
 
+  @Ignore String domainRepoId;
+
+  @Ignore String contactRepoId;
+
+  @Ignore String hostRepoId;
+
+  @Ignore Long domainRevisionId;
+
+  @Ignore Long contactRevisionId;
+
+  @Ignore Long hostRevisionId;
+
   public Key<HistoryEntry> getParentKey() {
     return parent;
   }
@@ -112,6 +151,9 @@ public abstract class PollMessage extends ImmutableObject
 
   public abstract ImmutableList<ResponseData> getResponseData();
 
+  @Override
+  public abstract VKey<? extends PollMessage> createVKey();
+
   /** Override Buildable.asBuilder() to give this method stronger typing. */
   @Override
   public abstract Builder<?, ?> asBuilder();
@@ -180,15 +222,83 @@ public abstract class PollMessage extends ImmutableObject
    * <p>One-time poll messages are deleted from Datastore once they have been delivered and ACKed.
    */
   @EntitySubclass(index = false)
+  @javax.persistence.Entity
+  @DiscriminatorValue("ONE_TIME")
+  @WithLongVKey
   public static class OneTime extends PollMessage {
 
     // Response data. Objectify cannot persist a base class type, so we must have a separate field
     // to hold every possible derived type of ResponseData that we might store.
+    @Transient
     List<ContactPendingActionNotificationResponse> contactPendingActionNotificationResponses;
-    List<ContactTransferResponse> contactTransferResponses;
+
+    @Transient List<ContactTransferResponse> contactTransferResponses;
+
+    @Transient
     List<DomainPendingActionNotificationResponse> domainPendingActionNotificationResponses;
-    List<DomainTransferResponse> domainTransferResponses;
-    List<HostPendingActionNotificationResponse> hostPendingActionNotificationResponses;
+
+    @Transient List<DomainTransferResponse> domainTransferResponses;
+
+    @Transient List<HostPendingActionNotificationResponse> hostPendingActionNotificationResponses;
+
+    @Ignore
+    @Embedded
+    @AttributeOverrides({
+      @AttributeOverride(
+          name = "nameOrId.value",
+          column = @Column(name = "pending_action_response_name_or_id")),
+      @AttributeOverride(
+          name = "nameOrId.actionResult",
+          column = @Column(name = "pending_action_response_action_result")),
+      @AttributeOverride(
+          name = "trid.serverTransactionId",
+          column = @Column(name = "pending_action_response_server_txn_id")),
+      @AttributeOverride(
+          name = "trid.clientTransactionId",
+          column = @Column(name = "pending_action_response_client_txn_id")),
+      @AttributeOverride(
+          name = "processedDate",
+          column = @Column(name = "pending_action_response_processed_date"))
+    })
+    PendingActionNotificationResponse pendingActionNotificationResponse;
+
+    @Ignore
+    @Embedded
+    @AttributeOverrides({
+      @AttributeOverride(
+          name = "transferStatus",
+          column = @Column(name = "transfer_response_transfer_status")),
+      @AttributeOverride(
+          name = "gainingClientId",
+          column = @Column(name = "transfer_response_gaining_registrar_id")),
+      @AttributeOverride(
+          name = "transferRequestTime",
+          column = @Column(name = "transfer_response_transfer_request_time")),
+      @AttributeOverride(
+          name = "losingClientId",
+          column = @Column(name = "transfer_response_losing_registrar_id")),
+      @AttributeOverride(
+          name = "pendingTransferExpirationTime",
+          column = @Column(name = "transfer_response_pending_transfer_expiration_time"))
+    })
+    TransferResponse transferResponse;
+
+    @Ignore
+    @Column(name = "transfer_response_domain_name")
+    String fullyQualifiedDomainName;
+
+    @Ignore
+    @Column(name = "transfer_response_domain_expiration_time")
+    DateTime extendedRegistrationExpirationTime;
+
+    @Ignore
+    @Column(name = "transfer_response_contact_id")
+    String contactId;
+
+    @Override
+    public VKey<OneTime> createVKey() {
+      return VKey.createOfy(this.getClass(), Key.create(this));
+    }
 
     @Override
     public Builder asBuilder() {
@@ -265,9 +375,13 @@ public abstract class PollMessage extends ImmutableObject
    * happens.
    */
   @EntitySubclass(index = false)
+  @javax.persistence.Entity
+  @DiscriminatorValue("AUTORENEW")
+  @WithLongVKey
   public static class Autorenew extends PollMessage {
 
     /** The target id of the autorenew event. */
+    @Column(name = "autorenew_domain_name")
     String targetId;
 
     /** The autorenew recurs annually between {@link #eventTime} and this time. */
@@ -282,6 +396,11 @@ public abstract class PollMessage extends ImmutableObject
       return autorenewEndTime;
     }
 
+    @Override
+    public VKey<Autorenew> createVKey() {
+      return VKey.createOfy(this.getClass(), Key.create(this));
+    }
+
     @Override
     public ImmutableList<ResponseData> getResponseData() {
       // Note that the event time is when the auto-renew occured, so the expiration time in the

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -236,7 +236,7 @@ public class Registrar extends ImmutableObject
    */
   @Id
   @javax.persistence.Id
-  @Column(name = "client_id", nullable = false)
+  @Column(name = "registrarId", nullable = false)
   String clientIdentifier;
 
   /**

core/src/main/java/google/registry/model/registry/Registry.java

@@ -579,6 +579,8 @@ public class Registry extends ImmutableObject implements Buildable {
     return Fee.create(
         eapFeeSchedule.getValueAtTime(now).getAmount(),
         FeeType.EAP,
+        // An EAP fee counts as premium so the domain's overall Fee doesn't show as standard-priced.
+        true,
         validPeriod,
         validPeriod.upperEndpoint());
   }

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -248,7 +248,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   }
 
   /**
-   * A premium list entry entity, persisted to Datastore.  Each instance represents the price of a
+   * A premium list entry entity, persisted to Datastore. Each instance represents the price of a
    * single label on a given TLD.
    */
   @ReportedOn

core/src/main/java/google/registry/model/registry/label/ReservedList.java

@@ -62,8 +62,8 @@ import org.joda.time.DateTime;
  */
 @Entity
 public final class ReservedList
-    extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry> implements
-    DatastoreEntity {
+    extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry>
+    implements DatastoreEntity {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -327,8 +327,8 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
   }
 
   /**
-   * Serves as the coordinating claims list singleton linking to the {@link ClaimsListRevision}
-   * that is live.
+   * Serves as the coordinating claims list singleton linking to the {@link ClaimsListRevision} that
+   * is live.
    */
   @Entity
   @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)

core/src/main/java/google/registry/model/transfer/BaseTransferObject.java

@@ -16,6 +16,9 @@ package google.registry.model.transfer;
 
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
+import javax.persistence.Column;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlTransient;
@@ -31,10 +34,12 @@ public abstract class BaseTransferObject extends ImmutableObject {
    * will always be non-null.
    */
   @XmlElement(name = "trStatus")
+  @Enumerated(EnumType.STRING)
   TransferStatus transferStatus;
 
   /** The gaining registrar of the current or last transfer. Can be null if never transferred. */
   @XmlElement(name = "reID")
+  @Column(name = "transfer_gaining_registrar_id")
   String gainingClientId;
 
   /** The time that the last transfer was requested. Can be null if never transferred. */
@@ -43,6 +48,7 @@ public abstract class BaseTransferObject extends ImmutableObject {
 
   /** The losing registrar of the current or last transfer. Can be null if never transferred. */
   @XmlElement(name = "acID")
+  @Column(name = "transfer_losing_registrar_id")
   String losingClientId;
 
   /**
@@ -51,6 +57,7 @@ public abstract class BaseTransferObject extends ImmutableObject {
    * this holds the time that the last pending transfer ended. Can be null if never transferred.
    */
   @XmlElement(name = "acDate")
+  @Column(name = "transfer_pending_expiration_time")
   DateTime pendingTransferExpirationTime;
 
   public TransferStatus getTransferStatus() {

core/src/main/java/google/registry/model/transfer/TransferData.java

@@ -17,8 +17,8 @@ package google.registry.model.transfer;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Unindex;
 import com.googlecode.objectify.condition.IfNull;
@@ -29,10 +29,14 @@ import google.registry.model.domain.Period;
 import google.registry.model.domain.Period.Unit;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.poll.PollMessage;
+import google.registry.persistence.VKey;
 import java.util.Set;
 import javax.annotation.Nullable;
-import javax.persistence.ElementCollection;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
 import javax.persistence.Embedded;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -47,7 +51,16 @@ public class TransferData extends BaseTransferObject implements Buildable {
   public static final TransferData EMPTY = new TransferData();
 
   /** The transaction id of the most recent transfer request (or null if there never was one). */
-  @Embedded Trid transferRequestTrid;
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "serverTransactionId",
+        column = @Column(name = "transfer_server_txn_id")),
+    @AttributeOverride(
+        name = "clientTransactionId",
+        column = @Column(name = "transfer_client_txn_id"))
+  })
+  Trid transferRequestTrid;
 
   /**
    * The period to extend the registration upon completion of the transfer.
@@ -55,23 +68,29 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * <p>By default, domain transfers are for one year. This can be changed to zero by using the
    * superuser EPP extension.
    */
-  @Embedded Period transferPeriod = Period.create(1, Unit.YEARS);
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "unit", column = @Column(name = "transfer_renew_period_unit")),
+    @AttributeOverride(name = "value", column = @Column(name = "transfer_renew_period_value"))
+  })
+  Period transferPeriod = Period.create(1, Unit.YEARS);
 
   /**
    * The registration expiration time resulting from the approval - speculative or actual - of the
    * most recent transfer request, applicable for domains only.
    *
    * <p>For pending transfers, this is the expiration time that will take effect under a projected
-   * server approval. For approved transfers, this is the actual expiration time of the domain as
-   * of the moment of transfer completion. For rejected or cancelled transfers, this field will be
+   * server approval. For approved transfers, this is the actual expiration time of the domain as of
+   * the moment of transfer completion. For rejected or cancelled transfers, this field will be
    * reset to null.
    *
    * <p>Note that even when this field is set, it does not necessarily mean that the post-transfer
-   * domain has a new expiration time.  Superuser transfers may not include a bundled 1 year renewal
+   * domain has a new expiration time. Superuser transfers may not include a bundled 1 year renewal
    * at all, or even when a renewal is bundled, for a transfer during the autorenew grace period the
    * bundled renewal simply subsumes the recent autorenewal, resulting in the same expiration time.
    */
   // TODO(b/36405140): backfill this field for existing domains to which it should apply.
+  @Column(name = "transfer_registration_expiration_time")
   DateTime transferredRegistrationExpirationTime;
 
   /**
@@ -83,18 +102,35 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * pending transfer is explicitly approved, rejected or cancelled, the referenced entities should
    * be deleted.
    */
+  @Transient
   @IgnoreSave(IfNull.class)
-  @ElementCollection
-  Set<Key<? extends TransferServerApproveEntity>> serverApproveEntities;
+  Set<VKey<? extends TransferServerApproveEntity>> serverApproveEntities;
+
+  // The following 3 fields are the replacement for serverApproveEntities in Cloud SQL.
+  // TODO(shicong): Add getter/setter for these 3 fields and use them in the application code.
+  @Ignore
+  @Column(name = "transfer_gaining_poll_message_id")
+  Long gainingTransferPollMessageId;
+
+  @Ignore
+  @Column(name = "transfer_losing_poll_message_id")
+  Long losingTransferPollMessageId;
+
+  @Ignore
+  @Column(name = "transfer_billing_cancellation_id")
+  Long billingCancellationId;
 
   /**
    * The regular one-time billing event that will be charged for a server-approved transfer.
    *
    * <p>This field should be null if there is not currently a pending transfer or if the object
    * being transferred is not a domain.
+   *
+   * <p>TODO(b/158230654) Remove unused columns for TransferData in Contact table.
    */
   @IgnoreSave(IfNull.class)
-  Key<BillingEvent.OneTime> serverApproveBillingEvent;
+  @Column(name = "transfer_billing_event_id")
+  VKey<BillingEvent.OneTime> serverApproveBillingEvent;
 
   /**
    * The autorenew billing event that should be associated with this resource after the transfer.
@@ -103,7 +139,8 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * being transferred is not a domain.
    */
   @IgnoreSave(IfNull.class)
-  Key<BillingEvent.Recurring> serverApproveAutorenewEvent;
+  @Column(name = "transfer_billing_recurrence_id")
+  VKey<BillingEvent.Recurring> serverApproveAutorenewEvent;
 
   /**
    * The autorenew poll message that should be associated with this resource after the transfer.
@@ -112,7 +149,8 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * being transferred is not a domain.
    */
   @IgnoreSave(IfNull.class)
-  Key<PollMessage.Autorenew> serverApproveAutorenewPollMessage;
+  @Column(name = "transfer_autorenew_poll_message_id")
+  VKey<PollMessage.Autorenew> serverApproveAutorenewPollMessage;
 
   @Nullable
   public Trid getTransferRequestTrid() {
@@ -128,22 +166,22 @@ public class TransferData extends BaseTransferObject implements Buildable {
     return transferredRegistrationExpirationTime;
   }
 
-  public ImmutableSet<Key<? extends TransferServerApproveEntity>> getServerApproveEntities() {
+  public ImmutableSet<VKey<? extends TransferServerApproveEntity>> getServerApproveEntities() {
     return nullToEmptyImmutableCopy(serverApproveEntities);
   }
 
   @Nullable
-  public Key<BillingEvent.OneTime> getServerApproveBillingEvent() {
+  public VKey<BillingEvent.OneTime> getServerApproveBillingEvent() {
     return serverApproveBillingEvent;
   }
 
   @Nullable
-  public Key<BillingEvent.Recurring> getServerApproveAutorenewEvent() {
+  public VKey<BillingEvent.Recurring> getServerApproveAutorenewEvent() {
     return serverApproveAutorenewEvent;
   }
 
   @Nullable
-  public Key<PollMessage.Autorenew> getServerApproveAutorenewPollMessage() {
+  public VKey<PollMessage.Autorenew> getServerApproveAutorenewPollMessage() {
     return serverApproveAutorenewPollMessage;
   }
 
@@ -202,25 +240,25 @@ public class TransferData extends BaseTransferObject implements Buildable {
     }
 
     public Builder setServerApproveEntities(
-        ImmutableSet<Key<? extends TransferServerApproveEntity>> serverApproveEntities) {
+        ImmutableSet<VKey<? extends TransferServerApproveEntity>> serverApproveEntities) {
       getInstance().serverApproveEntities = serverApproveEntities;
       return this;
     }
 
     public Builder setServerApproveBillingEvent(
-        Key<BillingEvent.OneTime> serverApproveBillingEvent) {
+        VKey<BillingEvent.OneTime> serverApproveBillingEvent) {
       getInstance().serverApproveBillingEvent = serverApproveBillingEvent;
       return this;
     }
 
     public Builder setServerApproveAutorenewEvent(
-        Key<BillingEvent.Recurring> serverApproveAutorenewEvent) {
+        VKey<BillingEvent.Recurring> serverApproveAutorenewEvent) {
       getInstance().serverApproveAutorenewEvent = serverApproveAutorenewEvent;
       return this;
     }
 
     public Builder setServerApproveAutorenewPollMessage(
-        Key<PollMessage.Autorenew> serverApproveAutorenewPollMessage) {
+        VKey<PollMessage.Autorenew> serverApproveAutorenewPollMessage) {
       getInstance().serverApproveAutorenewPollMessage = serverApproveAutorenewPollMessage;
       return this;
     }
@@ -230,5 +268,7 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * Marker interface for objects that are written in anticipation of a server approval, and
    * therefore need to be deleted under any other outcome.
    */
-  public interface TransferServerApproveEntity {}
+  public interface TransferServerApproveEntity {
+    VKey<? extends TransferServerApproveEntity> createVKey();
+  }
 }

core/src/main/java/google/registry/model/transfer/TransferResponse.java

@@ -17,6 +17,7 @@ package google.registry.model.transfer;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.EppResource;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
+import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
@@ -28,7 +29,8 @@ import org.joda.time.DateTime;
  * are common to all transfer responses; derived classes add resource specific fields.
  */
 @XmlTransient
-public abstract class TransferResponse extends BaseTransferObject implements ResponseData {
+@Embeddable
+public class TransferResponse extends BaseTransferObject implements ResponseData {
 
   /** An adapter to output the XML in response to a transfer command on a domain. */
   @Embed

core/src/main/java/google/registry/persistence/VKey.java

@@ -17,6 +17,7 @@ package google.registry.persistence;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.googlecode.objectify.Key;
 import google.registry.model.ImmutableObject;
 import java.util.Optional;
 
@@ -57,6 +58,23 @@ public class VKey<T> extends ImmutableObject {
     return new VKey(kind, ofyKey, null);
   }
 
+  /**
+   * Creates a {@link VKey} which only contains the ofy primary key by specifying the id of the
+   * {@link Key}.
+   */
+  public static <T> VKey<T> createOfy(Class<? extends T> kind, long id) {
+    return createOfy(kind, Key.create(kind, id));
+  }
+
+  /**
+   * Creates a {@link VKey} which only contains the ofy primary key by specifying the name of the
+   * {@link Key}.
+   */
+  public static <T> VKey<T> createOfy(Class<? extends T> kind, String name) {
+    checkArgumentNotNull(kind, "name must not be null");
+    return createOfy(kind, Key.create(kind, name));
+  }
+
   /** Creates a {@link VKey} which only contains both sql and ofy primary key. */
   public static <T> VKey<T> create(
       Class<? extends T> kind, Object sqlKey, com.googlecode.objectify.Key ofyKey) {

core/src/main/java/google/registry/persistence/converter/InetAddressSetConverter.java

@@ -0,0 +1,33 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import com.google.common.net.InetAddresses;
+import java.net.InetAddress;
+import javax.persistence.Converter;
+
+@Converter(autoApply = true)
+public class InetAddressSetConverter extends StringSetConverterBase<InetAddress> {
+
+  @Override
+  String toString(InetAddress element) {
+    return InetAddresses.toAddrString(element);
+  }
+
+  @Override
+  InetAddress fromString(String value) {
+    return InetAddresses.forString(value);
+  }
+}

core/src/main/java/google/registry/persistence/converter/TransferServerApproveEntitySetConverter.java

@@ -0,0 +1,37 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
+import google.registry.persistence.VKey;
+import java.util.Set;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** {@link AttributeConverter} for {@link Set}. */
+@Converter(autoApply = true)
+public class TransferServerApproveEntitySetConverter
+    extends StringSetConverterBase<VKey<? extends TransferServerApproveEntity>> {
+
+  @Override
+  String toString(VKey<? extends TransferServerApproveEntity> element) {
+    return String.valueOf(element.getSqlKey());
+  }
+
+  @Override
+  VKey<? extends TransferServerApproveEntity> fromString(String value) {
+    return VKey.createSql(TransferServerApproveEntity.class, Long.parseLong(value));
+  }
+}

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -278,7 +278,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
             .getResultList());
   }
 
-  private <T> int internalDelete(VKey<T> key) {
+  private int internalDelete(VKey<?> key) {
     checkArgumentNotNull(key, "key must be specified");
     assertInTransaction();
     EntityType<?> entityType = getEntityType(key.getKind());
@@ -291,10 +291,16 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public <T> void delete(VKey<T> key) {
+  public void delete(VKey<?> key) {
     internalDelete(key);
   }
 
+  @Override
+  public void delete(Iterable<? extends VKey<?>> vKeys) {
+    checkArgumentNotNull(vKeys, "vKeys must be specified");
+    vKeys.forEach(this::internalDelete);
+  }
+
   @Override
   public <T> void assertDelete(VKey<T> key) {
     if (internalDelete(key) != 1) {

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -115,7 +115,7 @@ public interface TransactionManager {
   <T> T load(VKey<T> key);
 
   /**
-   * Leads the set of entities by their key id.
+   * Loads the set of entities by their key id.
    *
    * @throws NoSuchElementException if any of the keys are not found.
    */
@@ -125,5 +125,8 @@ public interface TransactionManager {
   <T> ImmutableList<T> loadAll(Class<T> clazz);
 
   /** Deletes the entity by its id. */
-  <T> void delete(VKey<T> key);
+  void delete(VKey<?> key);
+
+  /** Deletes the set of entities by their key id. */
+  void delete(Iterable<? extends VKey<?>> keys);
 }

core/src/main/java/google/registry/rdap/RdapDomainSearchAction.java

@@ -14,6 +14,7 @@
 
 package google.registry.rdap;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -28,10 +29,10 @@ import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.InetAddresses;
 import com.google.common.primitives.Booleans;
-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.rdap.RdapJsonFormatter.OutputDataType;
 import google.registry.rdap.RdapMetrics.EndpointType;
 import google.registry.rdap.RdapMetrics.SearchType;
@@ -50,6 +51,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 import javax.inject.Inject;
 
 /**
@@ -243,7 +245,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
    */
   private DomainSearchResponse searchByNameserverLdhName(
       final RdapSearchPattern partialStringQuery) {
-    Iterable<Key<HostResource>> hostKeys = getNameserverRefsByLdhName(partialStringQuery);
+    Iterable<VKey<HostResource>> hostKeys = getNameserverRefsByLdhName(partialStringQuery);
     if (Iterables.isEmpty(hostKeys)) {
       metricInformationBuilder.setNumHostsRetrieved(0);
       throw new NotFoundException("No matching nameservers found");
@@ -261,7 +263,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
    * initial string is not required (e.g. "*.example.tld" is valid), because we can look up the
    * domain and just list all of its subordinate hosts.
    */
-  private Iterable<Key<HostResource>> getNameserverRefsByLdhName(
+  private Iterable<VKey<HostResource>> getNameserverRefsByLdhName(
       final RdapSearchPattern partialStringQuery) {
     // Handle queries without a wildcard.
     if (!partialStringQuery.getHasWildcard()) {
@@ -292,11 +294,13 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
     if (desiredRegistrar.isPresent()) {
       query = query.filter("currentSponsorClientId", desiredRegistrar.get());
     }
-    return query.keys();
+    return StreamSupport.stream(query.keys().spliterator(), false)
+        .map(key -> VKey.createOfy(HostResource.class, key))
+        .collect(toImmutableSet());
   }
 
   /** Assembles a list of {@link HostResource} keys by name when the pattern has no wildcard. */
-  private Iterable<Key<HostResource>> getNameserverRefsByLdhNameWithoutWildcard(
+  private Iterable<VKey<HostResource>> getNameserverRefsByLdhNameWithoutWildcard(
       final RdapSearchPattern partialStringQuery) {
     // If we need to check the sponsoring registrar, we need to load the resource rather than just
     // the key.
@@ -310,9 +314,9 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
       return (!host.isPresent()
               || !desiredRegistrar.get().equals(host.get().getPersistedCurrentSponsorClientId()))
           ? ImmutableList.of()
-          : ImmutableList.of(Key.create(host.get()));
+          : ImmutableList.of(host.get().createVKey());
     } else {
-      Key<HostResource> hostKey =
+      VKey<HostResource> hostKey =
           loadAndGetKey(
               HostResource.class,
               partialStringQuery.getInitialString(),
@@ -322,7 +326,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
   }
 
   /** Assembles a list of {@link HostResource} keys by name using a superordinate domain suffix. */
-  private Iterable<Key<HostResource>> getNameserverRefsByLdhNameWithSuffix(
+  private Iterable<VKey<HostResource>> getNameserverRefsByLdhNameWithSuffix(
       final RdapSearchPattern partialStringQuery) {
     // The suffix must be a domain that we manage. That way, we can look up the domain and search
     // through the subordinate hosts. This is more efficient, and lets us permit wildcard searches
@@ -338,7 +342,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
                         "A suffix in a lookup by nameserver name "
                             + "must be a domain defined in the system"));
     Optional<String> desiredRegistrar = getDesiredRegistrar();
-    ImmutableList.Builder<Key<HostResource>> builder = new ImmutableList.Builder<>();
+    ImmutableList.Builder<VKey<HostResource>> builder = new ImmutableList.Builder<>();
     for (String fqhn : ImmutableSortedSet.copyOf(domainBase.getSubordinateHosts())) {
       // We can't just check that the host name starts with the initial query string, because
       // then the query ns.exam*.example.com would match against nameserver ns.example.com.
@@ -351,10 +355,10 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
                   shouldIncludeDeleted() ? START_OF_TIME : getRequestTime());
           if (host.isPresent()
               && desiredRegistrar.get().equals(host.get().getPersistedCurrentSponsorClientId())) {
-            builder.add(Key.create(host.get()));
+            builder.add(host.get().createVKey());
           }
         } else {
-          Key<HostResource> hostKey =
+          VKey<HostResource> hostKey =
               loadAndGetKey(
                   HostResource.class,
                   fqhn,
@@ -400,7 +404,10 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
     if (desiredRegistrar.isPresent()) {
       query = query.filter("currentSponsorClientId", desiredRegistrar.get());
     }
-    return searchByNameserverRefs(query.keys());
+    return searchByNameserverRefs(
+        StreamSupport.stream(query.keys().spliterator(), false)
+            .map(key -> VKey.createOfy(HostResource.class, key))
+            .collect(toImmutableSet()));
   }
 
   /**
@@ -409,7 +416,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
    * <p>This method is called by {@link #searchByNameserverLdhName} and {@link
    * #searchByNameserverIp} after they assemble the relevant host keys.
    */
-  private DomainSearchResponse searchByNameserverRefs(final Iterable<Key<HostResource>> hostKeys) {
+  private DomainSearchResponse searchByNameserverRefs(final Iterable<VKey<HostResource>> hostKeys) {
     // We must break the query up into chunks, because the in operator is limited to 30 subqueries.
     // Since it is possible for the same domain to show up more than once in our result list (if
     // we do a wildcard nameserver search that returns multiple nameservers used by the same
@@ -420,11 +427,13 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
         ImmutableSortedSet.orderedBy(
             Comparator.comparing(DomainBase::getFullyQualifiedDomainName));
     int numHostKeysSearched = 0;
-    for (List<Key<HostResource>> chunk : Iterables.partition(hostKeys, 30)) {
+    for (List<VKey<HostResource>> chunk : Iterables.partition(hostKeys, 30)) {
       numHostKeysSearched += chunk.size();
-      Query<DomainBase> query = ofy().load()
-          .type(DomainBase.class)
-          .filter("nsHosts in", chunk);
+      Query<DomainBase> query =
+          ofy()
+              .load()
+              .type(DomainBase.class)
+              .filter("nsHosts in", chunk.stream().map(VKey::getOfyKey).collect(toImmutableSet()));
       if (!shouldIncludeDeleted()) {
         query = query.filter("deletionTime >", getRequestTime());
       // If we are not performing an inequality query, we can filter on the cursor in the query.

core/src/main/java/google/registry/rdap/RdapJsonFormatter.java

@@ -433,10 +433,7 @@ public class RdapJsonFormatter {
         statuses.add(StatusValue.LINKED);
       }
       if (hostResource.isSubordinate()
-          && ofy()
-              .load()
-              .key(hostResource.getSuperordinateDomain())
-              .now()
+          && tm().load(hostResource.getSuperordinateDomain())
               .cloneProjectedAtTime(getRequestTime())
               .getStatusValues()
               .contains(StatusValue.PENDING_TRANSFER)) {

core/src/main/java/google/registry/rde/HostResourceToXjcConverter.java

@@ -17,7 +17,6 @@ package google.registry.rde;
 import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.net.InetAddresses;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
@@ -35,9 +34,8 @@ import org.joda.time.DateTime;
 final class HostResourceToXjcConverter {
 
   /** Converts a subordinate {@link HostResource} to {@link XjcRdeHostElement}. */
-  static XjcRdeHostElement convertSubordinate(
-      HostResource host, DomainBase superordinateDomain) {
-    checkArgument(Key.create(superordinateDomain).equals(host.getSuperordinateDomain()));
+  static XjcRdeHostElement convertSubordinate(HostResource host, DomainBase superordinateDomain) {
+    checkArgument(superordinateDomain.createVKey().equals(host.getSuperordinateDomain()));
     return new XjcRdeHostElement(convertSubordinateHost(host, superordinateDomain));
   }
 

core/src/main/java/google/registry/rde/RdeStagingMapper.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Strings.nullToEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.EppResourceUtils.loadAtPointInTime;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.appengine.tools.mapreduce.Mapper;
 import com.google.auto.value.AutoValue;
@@ -186,13 +187,16 @@ public final class RdeStagingMapper extends Mapper<EppResource, PendingDeposit,
         return result;
       } else if (resource instanceof HostResource) {
         HostResource host = (HostResource) resource;
-        result = Optional.of(host.isSubordinate()
-            ? marshaller.marshalSubordinateHost(
-                host,
-                // Note that loadAtPointInTime() does cloneProjectedAtTime(watermark) for us.
-                loadAtPointInTime(
-                    ofy().load().key(host.getSuperordinateDomain()).now(), watermark).now())
-            : marshaller.marshalExternalHost(host));
+        result =
+            Optional.of(
+                host.isSubordinate()
+                    ? marshaller.marshalSubordinateHost(
+                        host,
+                        // Note that loadAtPointInTime() does cloneProjectedAtTime(watermark) for
+                        // us.
+                        loadAtPointInTime(tm().load(host.getSuperordinateDomain()), watermark)
+                            .now())
+                    : marshaller.marshalExternalHost(host));
         cache.put(WatermarkModePair.create(watermark, RdeMode.FULL), result);
         cache.put(WatermarkModePair.create(watermark, RdeMode.THIN), result);
         return result;

core/src/main/java/google/registry/tools/CommandUtilities.java

@@ -18,12 +18,12 @@ import static com.google.common.base.Preconditions.checkState;
 
 import com.google.common.base.Ascii;
 import com.google.common.base.Strings;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
+import google.registry.persistence.VKey;
 import org.joda.time.DateTime;
 
 /** Container class for static utility methods. */
@@ -41,7 +41,7 @@ class CommandUtilities {
       this.clazz = clazz;
     }
 
-    public Key<? extends EppResource> getKey(String uniqueId, DateTime now) {
+    public VKey<? extends EppResource> getKey(String uniqueId, DateTime now) {
       return ForeignKeyIndex.loadAndGetKey(clazz, uniqueId, now);
     }
   }

core/src/main/java/google/registry/tools/GetHistoryEntriesCommand.java

@@ -23,9 +23,9 @@ import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
 import google.registry.tools.CommandUtilities.ResourceType;
 import google.registry.xml.XmlTransformer;
 import org.joda.time.DateTime;
@@ -57,7 +57,7 @@ final class GetHistoryEntriesCommand implements CommandWithRemoteApi {
 
   @Override
   public void run() {
-    Key<? extends EppResource> parentKey = null;
+    VKey<? extends EppResource> parentKey = null;
     if (type != null || uniqueId != null) {
       checkArgument(
           type != null && uniqueId != null,
@@ -66,12 +66,12 @@ final class GetHistoryEntriesCommand implements CommandWithRemoteApi {
       checkArgumentNotNull(parentKey, "Invalid resource ID");
     }
     for (HistoryEntry entry :
-            (parentKey == null
+        (parentKey == null
                 ? ofy().load().type(HistoryEntry.class)
-                : ofy().load().type(HistoryEntry.class).ancestor(parentKey))
-        .order("modificationTime")
-        .filter("modificationTime >=", after)
-        .filter("modificationTime <=", before)) {
+                : ofy().load().type(HistoryEntry.class).ancestor(parentKey.getOfyKey()))
+            .order("modificationTime")
+            .filter("modificationTime >=", after)
+            .filter("modificationTime <=", before)) {
       System.out.printf(
           "Client: %s\nTime: %s\nClient TRID: %s\nServer TRID: %s\n%s\n",
           entry.getClientId(),

core/src/main/java/google/registry/tools/LevelDbLogReader.java

@@ -88,8 +88,8 @@ public final class LevelDbLogReader implements Iterator<byte[]> {
   }
 
   /**
-   * Returns the next {@link #BLOCK_SIZE} bytes from the input channel, or {@link
-   * Optional#empty()} if there is no more data.
+   * Returns the next {@link #BLOCK_SIZE} bytes from the input channel, or {@link Optional#empty()}
+   * if there is no more data.
    */
   // TODO(weiminyu): use ByteBuffer directly.
   private Optional<byte[]> readFromChannel() throws IOException {

core/src/main/java/google/registry/tools/ResaveEppResourceCommand.java

@@ -14,14 +14,14 @@
 
 package google.registry.tools;
 
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
+import google.registry.persistence.VKey;
 import google.registry.tools.CommandUtilities.ResourceType;
 import org.joda.time.DateTime;
 
@@ -48,12 +48,15 @@ public final class ResaveEppResourceCommand extends MutatingCommand {
 
   @Override
   protected void init() {
-    Key<? extends EppResource> resourceKey = checkArgumentNotNull(
-        type.getKey(uniqueId, DateTime.now(UTC)),
-        "Could not find active resource of type %s: %s", type, uniqueId);
+    VKey<? extends EppResource> resourceKey =
+        checkArgumentNotNull(
+            type.getKey(uniqueId, DateTime.now(UTC)),
+            "Could not find active resource of type %s: %s",
+            type,
+            uniqueId);
     // Load the resource directly to bypass running cloneProjectedAtTime() automatically, which can
     // cause stageEntityChange() to fail due to implicit projection changes.
-    EppResource resource = ofy().load().key(resourceKey).now();
+    EppResource resource = tm().load(resourceKey);
     stageEntityChange(resource, resource);
   }
 }

core/src/main/java/google/registry/ui/server/registrar/RegistryLockPostAction.java

@@ -30,7 +30,6 @@ import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.gson.Gson;
-import google.registry.config.RegistryConfig;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
@@ -49,12 +48,12 @@ import google.registry.tools.DomainLockUtils;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
 import java.net.URISyntaxException;
-import java.net.URL;
 import java.util.Map;
 import java.util.Optional;
 import javax.inject.Inject;
 import javax.mail.internet.AddressException;
 import javax.mail.internet.InternetAddress;
+import javax.servlet.http.HttpServletRequest;
 import org.apache.http.client.utils.URIBuilder;
 import org.joda.time.Duration;
 
@@ -76,11 +75,11 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final Gson GSON = new Gson();
 
-  private static final URL URL_BASE = RegistryConfig.getDefaultServer();
   private static final String VERIFICATION_EMAIL_TEMPLATE =
       "Please click the link below to perform the lock / unlock action on domain %s. Note: "
           + "this code will expire in one hour.\n\n%s";
 
+  private final HttpServletRequest req;
   private final JsonActionRunner jsonActionRunner;
   private final AuthResult authResult;
   private final AuthenticatedRegistrarAccessor registrarAccessor;
@@ -90,12 +89,14 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
 
   @Inject
   RegistryLockPostAction(
+      HttpServletRequest req,
       JsonActionRunner jsonActionRunner,
       AuthResult authResult,
       AuthenticatedRegistrarAccessor registrarAccessor,
       SendEmailService sendEmailService,
       DomainLockUtils domainLockUtils,
       @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress) {
+    this.req = req;
     this.jsonActionRunner = jsonActionRunner;
     this.authResult = authResult;
     this.registrarAccessor = registrarAccessor;
@@ -161,7 +162,7 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
       String url =
           new URIBuilder()
               .setScheme("https")
-              .setHost(URL_BASE.getHost())
+              .setHost(req.getServerName())
               .setPath("registry-lock-verify")
               .setParameter("lockVerificationCode", lock.getVerificationCode())
               .setParameter("isLock", String.valueOf(isLock))

core/src/main/java/google/registry/whois/DomainWhoisResponse.java

@@ -156,7 +156,7 @@ final class DomainWhoisResponse extends WhoisResponseImpl {
       // If we refer to a contact that doesn't exist, that's a bug. It means referential integrity
       // has somehow been broken. We skip the rest of this contact, but log it to hopefully bring it
       // someone's attention.
-      ContactResource contactResource = EppResource.loadCached(contact.get().getOfyKey());
+      ContactResource contactResource = EppResource.loadCached(contact.get());
       if (contactResource == null) {
         logger.atSevere().log(
             "(BUG) Broken reference found from domain %s to contact %s",

core/src/main/java/google/registry/whois/NameserverWhoisResponse.java

@@ -16,7 +16,7 @@ package google.registry.whois;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.net.InetAddresses;
@@ -49,7 +49,7 @@ final class NameserverWhoisResponse extends WhoisResponseImpl {
       HostResource host = hosts.get(i);
       String clientId =
           host.isSubordinate()
-              ? ofy().load().key(host.getSuperordinateDomain()).now()
+              ? tm().load(host.getSuperordinateDomain())
                   .cloneProjectedAtTime(getTimestamp())
                   .getCurrentSponsorClientId()
               : host.getPersistedCurrentSponsorClientId();

core/src/main/resources/META-INF/persistence.xml

@@ -36,6 +36,9 @@
     <class>google.registry.schema.tld.ReservedList</class>
     <class>google.registry.model.domain.secdns.DelegationSignerData</class>
     <class>google.registry.model.domain.GracePeriod</class>
+    <class>google.registry.model.poll.PollMessage</class>
+    <class>google.registry.model.poll.PollMessage$OneTime</class>
+    <class>google.registry.model.poll.PollMessage$Autorenew</class>
 
     <!-- Customized type converters -->
     <class>google.registry.persistence.converter.BillingCostTransitionConverter</class>
@@ -47,20 +50,25 @@
     <class>google.registry.persistence.converter.CurrencyUnitConverter</class>
     <class>google.registry.persistence.converter.DateTimeConverter</class>
     <class>google.registry.persistence.converter.DurationConverter</class>
+    <class>google.registry.persistence.converter.InetAddressSetConverter</class>
     <class>google.registry.persistence.converter.PostalInfoChoiceListConverter</class>
     <class>google.registry.persistence.converter.RegistrarPocSetConverter</class>
     <class>google.registry.persistence.converter.StatusValueSetConverter</class>
     <class>google.registry.persistence.converter.StringListConverter</class>
     <class>google.registry.persistence.converter.StringSetConverter</class>
     <class>google.registry.persistence.converter.TldStateTransitionConverter</class>
+    <class>google.registry.persistence.converter.TransferServerApproveEntitySetConverter</class>
     <class>google.registry.persistence.converter.UpdateAutoTimestampConverter</class>
     <class>google.registry.persistence.converter.ZonedDateTimeConverter</class>
 
     <!-- Generated converters for VKey -->
     <class>google.registry.model.billing.VKeyConverter_BillingEvent</class>
+    <class>google.registry.model.domain.VKeyConverter_DomainBase</class>
+    <class>google.registry.model.contact.VKeyConverter_ContactResource</class>
     <class>google.registry.model.domain.token.VKeyConverter_AllocationToken</class>
     <class>google.registry.model.host.VKeyConverter_HostResource</class>
-    <class>google.registry.model.contact.VKeyConverter_ContactResource</class>
+    <class>google.registry.model.poll.VKeyConverter_Autorenew</class>
+    <class>google.registry.model.poll.VKeyConverter_OneTime</class>
 
     <!-- TODO(weiminyu): check out application-layer validation. -->
     <validation-mode>NONE</validation-mode>

core/src/test/java/google/registry/backup/CommitLogExports.java

@@ -0,0 +1,231 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Lists.partition;
+import static google.registry.backup.BackupUtils.serializeEntity;
+import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static google.registry.util.DateTimeUtils.isAtOrAfter;
+import static java.util.Comparator.comparingLong;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Streams;
+import com.googlecode.objectify.Key;
+import google.registry.model.ImmutableObject;
+import google.registry.model.ofy.CommitLogBucket;
+import google.registry.model.ofy.CommitLogCheckpoint;
+import google.registry.model.ofy.CommitLogCheckpointRoot;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import google.registry.util.Clock;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import org.joda.time.DateTime;
+
+/**
+ * Helpers for exporting the diff between two commit log checkpoints to a local file.
+ *
+ * <p>In production, CommitLogs are saved periodically by cron jobs. During each job, the {@link
+ * CommitLogCheckpointAction} is invoked first to compute a {@link CommitLogCheckpoint} and persist
+ * it in Datastore. Then the {@link ExportCommitLogDiffAction} is invoked to export the diffs
+ * accumulated between the previous and current checkpoints to a file.
+ *
+ * <p>The {@link #computeCheckpoint(Clock)} method is copied with simplification from {@link
+ * CommitLogCheckpointAction}, and the {@link #saveCommitLogs(String, CommitLogCheckpoint,
+ * CommitLogCheckpoint)} method is copied with simplification from {@link
+ * ExportCommitLogDiffAction}. We opted for copying instead of refactoring to reduce risk to
+ * production code.
+ */
+public final class CommitLogExports {
+
+  public static final String DIFF_FILE_PREFIX = "commit_diff_until_";
+
+  private static final int EXPORT_DIFF_BATCH_SIZE = 100;
+
+  private CommitLogExports() {}
+
+  /**
+   * Returns the next {@link CommitLogCheckpoint} for Commit logs. Please refer to the class javadoc
+   * for background.
+   */
+  public static CommitLogCheckpoint computeCheckpoint(Clock clock) {
+    CommitLogCheckpointStrategy strategy = new CommitLogCheckpointStrategy();
+    strategy.clock = clock;
+    strategy.ofy = ofy();
+
+    CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
+    tm().transact(
+            () -> {
+              DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
+              checkState(
+                  checkpoint.getCheckpointTime().isAfter(lastWrittenTime),
+                  "Newer checkpoint already written at time: %s",
+                  lastWrittenTime);
+              ofy()
+                  .saveWithoutBackup()
+                  .entities(
+                      checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
+            });
+    return checkpoint;
+  }
+
+  /**
+   * Saves the incremental changes between {@code prevCheckpoint} and {@code checkpoint} and returns
+   * the {@link File}. Please refer to class javadoc for background.
+   */
+  public static File saveCommitLogs(
+      String commitLogDir,
+      @Nullable CommitLogCheckpoint prevCheckpoint,
+      CommitLogCheckpoint checkpoint) {
+    checkArgument(
+        prevCheckpoint == null
+            || (isAtOrAfter(prevCheckpoint.getCheckpointTime(), START_OF_TIME)
+                && prevCheckpoint.getCheckpointTime().isBefore(checkpoint.getCheckpointTime())),
+        "Inversed checkpoint: prev is %s, current is %s.",
+        Optional.ofNullable(prevCheckpoint)
+            .map(CommitLogCheckpoint::getCheckpointTime)
+            .map(DateTime::toString)
+            .orElse("null"),
+        checkpoint.getCheckpointTime().toString());
+
+    // Load the keys of all the manifests to include in this diff.
+    List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(prevCheckpoint, checkpoint);
+    // Open an output channel to GCS, wrapped in a stream for convenience.
+    File commitLogFile =
+        new File(commitLogDir + "/" + DIFF_FILE_PREFIX + checkpoint.getCheckpointTime());
+    try (OutputStream commitLogStream =
+        new BufferedOutputStream(new FileOutputStream(commitLogFile))) {
+      // Export the upper checkpoint itself.
+      serializeEntity(checkpoint, commitLogStream);
+      // If there are no manifests to export, stop early, now that we've written out the file with
+      // the checkpoint itself (which is needed for restores, even if it's empty).
+      if (sortedKeys.isEmpty()) {
+        return commitLogFile;
+      }
+      // Export to GCS in chunks, one per fixed batch of commit logs.  While processing one batch,
+      // asynchronously load the entities for the next one.
+      List<List<Key<CommitLogManifest>>> keyChunks = partition(sortedKeys, EXPORT_DIFF_BATCH_SIZE);
+      // Objectify's map return type is asynchronous. Calling .values() will block until it loads.
+      Map<?, CommitLogManifest> nextChunkToExport = ofy().load().keys(keyChunks.get(0));
+      for (int i = 0; i < keyChunks.size(); i++) {
+        // Force the async load to finish.
+        Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
+        // Since there is no hard bound on how much data this might be, take care not to let the
+        // Objectify session cache fill up and potentially run out of memory. This is the only safe
+        // point to do this since at this point there is no async load in progress.
+        ofy().clearSessionCache();
+        // Kick off the next async load, which can happen in parallel to the current GCS export.
+        if (i + 1 < keyChunks.size()) {
+          nextChunkToExport = ofy().load().keys(keyChunks.get(i + 1));
+        }
+        exportChunk(commitLogStream, chunkValues);
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return commitLogFile;
+  }
+
+  /**
+   * Loads all the diff keys, sorted in a transaction-consistent chronological order.
+   *
+   * @param lowerCheckpoint exclusive lower bound on keys in this diff, or null if no lower bound
+   * @param upperCheckpoint inclusive upper bound on keys in this diff
+   */
+  private static ImmutableList<Key<CommitLogManifest>> loadAllDiffKeys(
+      @Nullable final CommitLogCheckpoint lowerCheckpoint,
+      final CommitLogCheckpoint upperCheckpoint) {
+    // Fetch the keys (no data) between these checkpoints, and sort by timestamp. This ordering is
+    // transaction-consistent by virtue of our checkpoint strategy and our customized Ofy; see
+    // CommitLogCheckpointStrategy for the proof. We break ties by sorting on bucket ID to ensure
+    // a deterministic order.
+    return upperCheckpoint.getBucketTimestamps().keySet().stream()
+        .flatMap(
+            bucketNum ->
+                Streams.stream(loadDiffKeysFromBucket(lowerCheckpoint, upperCheckpoint, bucketNum)))
+        .sorted(
+            comparingLong(Key<CommitLogManifest>::getId)
+                .thenComparingLong(a -> a.getParent().getId()))
+        .collect(toImmutableList());
+  }
+
+  /**
+   * Loads the diff keys for one bucket.
+   *
+   * @param lowerCheckpoint exclusive lower bound on keys in this diff, or null if no lower bound
+   * @param upperCheckpoint inclusive upper bound on keys in this diff
+   * @param bucketNum the bucket to load diff keys from
+   */
+  private static Iterable<Key<CommitLogManifest>> loadDiffKeysFromBucket(
+      @Nullable CommitLogCheckpoint lowerCheckpoint,
+      CommitLogCheckpoint upperCheckpoint,
+      int bucketNum) {
+    // If no lower checkpoint exists, or if it exists but had no timestamp for this bucket number
+    // (because the bucket count was increased between these checkpoints), then use START_OF_TIME
+    // as the effective exclusive lower bound.
+    DateTime lowerCheckpointBucketTime =
+        firstNonNull(
+            (lowerCheckpoint == null) ? null : lowerCheckpoint.getBucketTimestamps().get(bucketNum),
+            START_OF_TIME);
+    // Since START_OF_TIME=0 is not a valid id in a key, add 1 to both bounds. Then instead of
+    // loading lowerBound < x <= upperBound, we can load lowerBound <= x < upperBound.
+    DateTime lowerBound = lowerCheckpointBucketTime.plusMillis(1);
+    DateTime upperBound = upperCheckpoint.getBucketTimestamps().get(bucketNum).plusMillis(1);
+    // If the lower and upper bounds are equal, there can't be any results, so skip the query.
+    if (lowerBound.equals(upperBound)) {
+      return ImmutableSet.of();
+    }
+    Key<CommitLogBucket> bucketKey = getBucketKey(bucketNum);
+    return ofy()
+        .load()
+        .type(CommitLogManifest.class)
+        .ancestor(bucketKey)
+        .filterKey(">=", CommitLogManifest.createKey(bucketKey, lowerBound))
+        .filterKey("<", CommitLogManifest.createKey(bucketKey, upperBound))
+        .keys();
+  }
+
+  /** Writes a chunks-worth of manifests and associated mutations to GCS. */
+  private static void exportChunk(OutputStream gcsStream, Collection<CommitLogManifest> chunk)
+      throws IOException {
+    // Kickoff async loads for all the manifests in the chunk.
+    ImmutableList.Builder<Iterable<? extends ImmutableObject>> entities =
+        new ImmutableList.Builder<>();
+    for (CommitLogManifest manifest : chunk) {
+      entities.add(ImmutableList.of(manifest));
+      entities.add(ofy().load().type(CommitLogMutation.class).ancestor(manifest));
+    }
+    for (ImmutableObject entity : concat(entities.build())) {
+      serializeEntity(entity, gcsStream);
+    }
+  }
+}

core/src/test/java/google/registry/batch/DeleteContactsAndHostsActionTest.java

@@ -609,7 +609,7 @@ public class DeleteContactsAndHostsActionTest
         .hasDeletionTime(END_OF_TIME);
     DomainBase domain =
         loadByForeignKey(DomainBase.class, "example.tld", clock.nowUtc()).get();
-    assertThat(domain.getNameservers()).contains(hostAfter.createKey());
+    assertThat(domain.getNameservers()).contains(hostAfter.createVKey());
     HistoryEntry historyEntry = getOnlyHistoryEntryOfType(hostAfter, HOST_DELETE_FAILURE);
     assertPollMessageFor(
         historyEntry,
@@ -679,7 +679,7 @@ public class DeleteContactsAndHostsActionTest
     persistResource(
         newDomainBase("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .setDeletionTime(clock.nowUtc().minusDays(5))
             .build());
     enqueuer.enqueueAsyncDelete(
@@ -725,7 +725,7 @@ public class DeleteContactsAndHostsActionTest
         persistResource(
             persistHostPendingDelete("ns2.example.tld")
                 .asBuilder()
-                .setSuperordinateDomain(Key.create(domain))
+                .setSuperordinateDomain(domain.createVKey())
                 .build());
     enqueuer.enqueueAsyncDelete(
         host,
@@ -938,7 +938,7 @@ public class DeleteContactsAndHostsActionTest
     return persistResource(
         newDomainBase(domainName, contact)
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .build());
   }
 

core/src/test/java/google/registry/beam/initsql/BackupTestStore.java

@@ -0,0 +1,160 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.google.appengine.api.datastore.Entity;
+import com.googlecode.objectify.Key;
+import google.registry.backup.CommitLogExports;
+import google.registry.model.ofy.CommitLogCheckpoint;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.FakeClock;
+import google.registry.tools.LevelDbFileBuilder;
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/**
+ * Wrapper of a Datastore test instance that can generate backups.
+ *
+ * <p>A Datastore backup consists of an unsynchronized data export and a sequence of incremental
+ * Commit Logs that overlap with the export process. Together they can be used to recreate a
+ * consistent snapshot of the Datastore.
+ *
+ * <p>For convenience of test-writing, the {@link #fakeClock} is advanced by 1 millisecond after
+ * every transaction is invoked on this store, ensuring strictly increasing timestamps on causally
+ * dependent transactions. In production, the same ordering is ensured by sleep and retry.
+ */
+class BackupTestStore implements AutoCloseable {
+
+  private static final DateTimeFormatter EXPORT_TIMESTAMP_FORMAT =
+      DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss_SSS");
+
+  private final FakeClock fakeClock;
+  private AppEngineRule appEngine;
+
+  private CommitLogCheckpoint prevCommitLogCheckpoint;
+
+  BackupTestStore(FakeClock fakeClock) throws Exception {
+    this.fakeClock = fakeClock;
+    this.appEngine =
+        new AppEngineRule.Builder()
+            .withDatastore()
+            .withoutCannedData()
+            .withClock(fakeClock)
+            .build();
+    this.appEngine.beforeEach(null);
+  }
+
+  void transact(Iterable<Object> deletes, Iterable<Object> newOrUpdated) {
+    tm().transact(
+            () -> {
+              ofy().delete().entities(deletes);
+              ofy().save().entities(newOrUpdated);
+            });
+    fakeClock.advanceOneMilli();
+  }
+
+  /** Inserts or updates {@code entities} in the Datastore. */
+  @SafeVarargs
+  final void insertOrUpdate(Object... entities) {
+    tm().transact(() -> ofy().save().entities(entities).now());
+    fakeClock.advanceOneMilli();
+  }
+
+  /** Deletes {@code entities} from the Datastore. */
+  @SafeVarargs
+  final void delete(Object... entities) {
+    tm().transact(() -> ofy().delete().entities(entities).now());
+    fakeClock.advanceOneMilli();
+  }
+
+  /**
+   * Exports entities of the caller provided types and returns the directory where data is exported.
+   *
+   * @param exportRootPath path to the root directory of all exports. A subdirectory will be created
+   *     for this export
+   * @param pojoTypes java class of all entities to be exported
+   * @param excludes {@link Set} of {@link Key keys} of the entities not to export.This can be used
+   *     to simulate an inconsistent export
+   * @return directory where data is exported
+   */
+  File export(String exportRootPath, Iterable<Class<?>> pojoTypes, Set<Key<?>> excludes)
+      throws IOException {
+    File exportDirectory = getExportDirectory(exportRootPath);
+    for (Class<?> pojoType : pojoTypes) {
+      File perKindFile =
+          new File(
+              BackupPaths.getExportFileNameByShard(
+                  exportDirectory.getAbsolutePath(), Key.getKind(pojoType), 0));
+      checkState(
+          perKindFile.getParentFile().mkdirs(),
+          "Failed to create per-kind export directory for %s.",
+          perKindFile.getParentFile().getAbsolutePath());
+      exportOneKind(perKindFile, pojoType, excludes);
+    }
+    return exportDirectory;
+  }
+
+  private void exportOneKind(File perKindFile, Class<?> pojoType, Set<Key<?>> excludes)
+      throws IOException {
+    LevelDbFileBuilder builder = new LevelDbFileBuilder(perKindFile);
+    for (Object pojo : ofy().load().type(pojoType).iterable()) {
+      if (!excludes.contains(Key.create(pojo))) {
+        try {
+          builder.addEntity(toEntity(pojo));
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+    builder.build();
+  }
+
+  File saveCommitLogs(String commitLogDir) {
+    CommitLogCheckpoint checkpoint = CommitLogExports.computeCheckpoint(fakeClock);
+    File commitLogFile =
+        CommitLogExports.saveCommitLogs(commitLogDir, prevCommitLogCheckpoint, checkpoint);
+    prevCommitLogCheckpoint = checkpoint;
+    return commitLogFile;
+  }
+
+  @Override
+  public void close() throws Exception {
+    if (appEngine != null) {
+      appEngine.afterEach(null);
+      appEngine = null;
+    }
+  }
+
+  private Entity toEntity(Object pojo) {
+    return tm().transactNew(() -> ofy().save().toEntity(pojo));
+  }
+
+  private File getExportDirectory(String exportRootPath) {
+    File exportDirectory =
+        new File(exportRootPath, fakeClock.nowUtc().toString(EXPORT_TIMESTAMP_FORMAT));
+    checkState(
+        exportDirectory.mkdirs(),
+        "Failed to create export directory %s.",
+        exportDirectory.getAbsolutePath());
+    return exportDirectory;
+  }
+}

core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java

@@ -0,0 +1,262 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatastoreHelper.newContactResource;
+import static google.registry.testing.DatastoreHelper.newDomainBase;
+import static google.registry.testing.DatastoreHelper.newRegistry;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Streams;
+import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
+import com.googlecode.objectify.Key;
+import google.registry.backup.CommitLogImports;
+import google.registry.backup.VersionedEntity;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DesignatedContact;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.Ofy;
+import google.registry.model.registry.Registry;
+import google.registry.persistence.VKey;
+import google.registry.testing.FakeClock;
+import google.registry.testing.InjectRule;
+import google.registry.tools.LevelDbLogReader;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Stream;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.io.TempDir;
+
+/** Unit tests for {@link BackupTestStore}. */
+public class BackupTestStoreTest {
+  private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z");
+
+  private FakeClock fakeClock;
+  private BackupTestStore store;
+
+  private Registry registry;
+  private ContactResource contact;
+  private DomainBase domain;
+
+  @TempDir File tempDir;
+
+  @RegisterExtension InjectRule injectRule = new InjectRule();
+
+  @BeforeEach
+  void beforeEach() throws Exception {
+    fakeClock = new FakeClock(START_TIME);
+    store = new BackupTestStore(fakeClock);
+    injectRule.setStaticField(Ofy.class, "clock", fakeClock);
+
+    registry = newRegistry("tld1", "TLD1");
+    store.insertOrUpdate(registry);
+    contact = newContactResource("contact_1");
+    domain = newDomainBase("domain1.tld1", contact);
+    store.insertOrUpdate(contact, domain);
+  }
+
+  @AfterEach
+  void afterEach() throws Exception {
+    store.close();
+  }
+
+  @Test
+  void export_filesCreated() throws IOException {
+    String exportRootPath = tempDir.getAbsolutePath();
+    assertThat(fakeClock.nowUtc().toString()).isEqualTo("2000-01-01T00:00:00.002Z");
+    File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_002");
+    assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile())
+        .that(exportFolder.exists())
+        .isFalse();
+    File actualExportFolder = export(exportRootPath, Collections.EMPTY_SET);
+    assertThat(actualExportFolder).isEquivalentAccordingToCompareTo(exportFolder);
+    try (Stream<String> files =
+        Files.walk(exportFolder.toPath())
+            .filter(Files::isRegularFile)
+            .map(Path::toString)
+            .map(string -> string.substring(exportFolder.getAbsolutePath().length()))) {
+      assertThat(files)
+          .containsExactly(
+              "/all_namespaces/kind_Registry/input-0",
+              "/all_namespaces/kind_DomainBase/input-0",
+              "/all_namespaces/kind_ContactResource/input-0");
+    }
+  }
+
+  @Test
+  void export_folderNameChangesWithTime() throws IOException {
+    String exportRootPath = tempDir.getAbsolutePath();
+    fakeClock.advanceOneMilli();
+    File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_003");
+    assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile())
+        .that(exportFolder.exists())
+        .isFalse();
+    assertThat(export(exportRootPath, Collections.EMPTY_SET))
+        .isEquivalentAccordingToCompareTo(exportFolder);
+  }
+
+  @Test
+  void export_dataReadBack() throws IOException {
+    String exportRootPath = tempDir.getAbsolutePath();
+    File exportFolder = export(exportRootPath, Collections.EMPTY_SET);
+    ImmutableList<String> tldStrings =
+        loadPropertyFromExportedEntities(
+            new File(exportFolder, "/all_namespaces/kind_Registry/input-0"),
+            Registry.class,
+            Registry::getTldStr);
+    assertThat(tldStrings).containsExactly("tld1");
+    ImmutableList<String> domainStrings =
+        loadPropertyFromExportedEntities(
+            new File(exportFolder, "/all_namespaces/kind_DomainBase/input-0"),
+            DomainBase.class,
+            DomainBase::getFullyQualifiedDomainName);
+    assertThat(domainStrings).containsExactly("domain1.tld1");
+    ImmutableList<String> contactIds =
+        loadPropertyFromExportedEntities(
+            new File(exportFolder, "/all_namespaces/kind_ContactResource/input-0"),
+            ContactResource.class,
+            ContactResource::getContactId);
+    assertThat(contactIds).containsExactly("contact_1");
+  }
+
+  @Test
+  void export_excludeSomeEntity() throws IOException {
+    store.insertOrUpdate(newRegistry("tld2", "TLD2"));
+    String exportRootPath = tempDir.getAbsolutePath();
+    File exportFolder =
+        export(
+            exportRootPath, ImmutableSet.of(Key.create(getCrossTldKey(), Registry.class, "tld1")));
+    ImmutableList<String> tlds =
+        loadPropertyFromExportedEntities(
+            new File(exportFolder, "/all_namespaces/kind_Registry/input-0"),
+            Registry.class,
+            Registry::getTldStr);
+    assertThat(tlds).containsExactly("tld2");
+  }
+
+  @Test
+  void saveCommitLogs_fileCreated() {
+    File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath());
+    assertThat(commitLogFile.exists()).isTrue();
+    assertThat(commitLogFile.getName()).isEqualTo("commit_diff_until_2000-01-01T00:00:00.002Z");
+  }
+
+  @Test
+  void saveCommitLogs_inserts() {
+    File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath());
+    assertThat(commitLogFile.exists()).isTrue();
+    ImmutableList<VersionedEntity> mutations = CommitLogImports.loadEntities(commitLogFile);
+    assertThat(mutations.stream().map(VersionedEntity::getEntity).map(Optional::get))
+        .containsExactlyElementsIn(toDatastoreEntities(registry, contact, domain));
+    // Registry created at -2, contract and domain created at -1.
+    assertThat(mutations.stream().map(VersionedEntity::commitTimeMills))
+        .containsExactly(
+            fakeClock.nowUtc().getMillis() - 2,
+            fakeClock.nowUtc().getMillis() - 1,
+            fakeClock.nowUtc().getMillis() - 1);
+  }
+
+  @Test
+  void saveCommitLogs_deletes() {
+    fakeClock.advanceOneMilli();
+    store.saveCommitLogs(tempDir.getAbsolutePath());
+    ContactResource newContact = newContactResource("contact2");
+    VKey<ContactResource> vKey = newContact.createVKey();
+    domain =
+        domain
+            .asBuilder()
+            .setRegistrant(vKey)
+            .setContacts(
+                ImmutableSet.of(
+                    DesignatedContact.create(DesignatedContact.Type.ADMIN, vKey),
+                    DesignatedContact.create(DesignatedContact.Type.TECH, vKey)))
+            .build();
+    store.insertOrUpdate(domain, newContact);
+    store.delete(contact);
+    fakeClock.advanceOneMilli();
+    File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath());
+    ImmutableList<VersionedEntity> mutations = CommitLogImports.loadEntities(commitLogFile);
+    assertThat(mutations.stream().filter(VersionedEntity::isDelete).map(VersionedEntity::key))
+        .containsExactly(Key.create(contact).getRaw());
+
+    assertThat(
+            mutations.stream()
+                .filter(Predicates.not(VersionedEntity::isDelete))
+                .map(VersionedEntity::getEntity)
+                .map(Optional::get))
+        .containsExactlyElementsIn(toDatastoreEntities(domain, newContact));
+  }
+
+  @Test
+  void saveCommitLogs_empty() {
+    fakeClock.advanceOneMilli();
+    store.saveCommitLogs(tempDir.getAbsolutePath());
+    fakeClock.advanceOneMilli();
+    File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath());
+    assertThat(commitLogFile.exists()).isTrue();
+    assertThat(CommitLogImports.loadEntities(commitLogFile)).isEmpty();
+  }
+
+  private File export(String exportRootPath, Set<Key<?>> excludes) throws IOException {
+    return store.export(
+        exportRootPath,
+        ImmutableList.of(ContactResource.class, DomainBase.class, Registry.class),
+        excludes);
+  }
+
+  private static <T> ImmutableList<String> loadPropertyFromExportedEntities(
+      File dataFile, Class<T> ofyEntityType, Function<T, String> getter) throws IOException {
+    return Streams.stream(LevelDbLogReader.from(dataFile.toPath()))
+        .map(bytes -> toOfyEntity(bytes, ofyEntityType))
+        .map(getter)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  private static <T> T toOfyEntity(byte[] rawRecord, Class<T> ofyEntityType) {
+    EntityProto proto = new EntityProto();
+    proto.parseFrom(rawRecord);
+    Entity entity = EntityTranslator.createFromPb(proto);
+    return ofyEntityType.cast(ofy().load().fromEntity(entity));
+  }
+
+  @SafeVarargs
+  private static ImmutableList<Entity> toDatastoreEntities(Object... ofyEntities) {
+    return tm().transact(
+            () ->
+                Stream.of(ofyEntities)
+                    .map(oe -> ofy().save().toEntity(oe))
+                    .collect(ImmutableList.toImmutableList()));
+  }
+}

core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java

@@ -0,0 +1,223 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatastoreHelper.newContactResource;
+import static google.registry.testing.DatastoreHelper.newDomainBase;
+import static google.registry.testing.DatastoreHelper.newRegistry;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.common.collect.ImmutableList;
+import google.registry.backup.VersionedEntity;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.Ofy;
+import google.registry.model.registry.Registry;
+import google.registry.testing.FakeClock;
+import google.registry.testing.InjectRule;
+import java.io.File;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.DateTime;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link CommitLogTransforms}. */
+// TODO(weiminyu): Upgrade to JUnit5 when TestPipeline is upgraded. It is also easy to adapt with
+// a wrapper.
+@RunWith(JUnit4.class)
+public class CommitLogTransformsTest implements Serializable {
+  private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z");
+
+  @Rule public final transient TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  @Rule public final transient InjectRule injectRule = new InjectRule();
+
+  @Rule
+  public final transient TestPipeline pipeline =
+      TestPipeline.create().enableAbandonedNodeEnforcement(true);
+
+  private FakeClock fakeClock;
+  private transient BackupTestStore store;
+  private File commitLogsDir;
+  private File firstCommitLogFile;
+  // Canned data that are persisted to Datastore, used by assertions in tests.
+  // TODO(weiminyu): use Ofy entity pojos directly.
+  private transient ImmutableList<Entity> persistedEntities;
+
+  @Before
+  public void beforeEach() throws Exception {
+    fakeClock = new FakeClock(START_TIME);
+    store = new BackupTestStore(fakeClock);
+    injectRule.setStaticField(Ofy.class, "clock", fakeClock);
+
+    Registry registry = newRegistry("tld1", "TLD1");
+    store.insertOrUpdate(registry);
+    ContactResource contact1 = newContactResource("contact_1");
+    DomainBase domain1 = newDomainBase("domain1.tld1", contact1);
+    store.insertOrUpdate(contact1, domain1);
+    persistedEntities =
+        ImmutableList.of(registry, contact1, domain1).stream()
+            .map(ofyEntity -> tm().transact(() -> ofy().save().toEntity(ofyEntity)))
+            .collect(ImmutableList.toImmutableList());
+    commitLogsDir = temporaryFolder.newFolder();
+    firstCommitLogFile = store.saveCommitLogs(commitLogsDir.getAbsolutePath());
+  }
+
+  @After
+  public void afterEach() throws Exception {
+    if (store != null) {
+      store.close();
+      store = null;
+    }
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void getCommitLogFilePatterns() {
+    PCollection<String> patterns =
+        pipeline.apply(
+            "Get CommitLog file patterns",
+            CommitLogTransforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath()));
+
+    ImmutableList<String> expectedPatterns =
+        ImmutableList.of(commitLogsDir.getAbsolutePath() + "/commit_diff_until_*");
+
+    PAssert.that(patterns).containsInAnyOrder(expectedPatterns);
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void getFilesByPatterns() {
+    PCollection<Metadata> fileMetas =
+        pipeline
+            .apply(
+                "File patterns to metadata",
+                Create.of(commitLogsDir.getAbsolutePath() + "/commit_diff_until_*")
+                    .withCoder(StringUtf8Coder.of()))
+            .apply(Transforms.getFilesByPatterns());
+
+    // Transform fileMetas to file names for assertions.
+    PCollection<String> fileNames =
+        fileMetas.apply(
+            "File metadata to path string",
+            ParDo.of(
+                new DoFn<Metadata, String>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element Metadata metadata, OutputReceiver<String> out) {
+                    out.output(metadata.resourceId().toString());
+                  }
+                }));
+
+    ImmutableList<String> expectedFilenames =
+        ImmutableList.of(firstCommitLogFile.getAbsolutePath());
+
+    PAssert.that(fileNames).containsInAnyOrder(expectedFilenames);
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void filterCommitLogsByTime() {
+    ImmutableList<String> commitLogFilenames =
+        ImmutableList.of(
+            "/commit_diff_until_2000-01-01T00:00:00.000Z",
+            "/commit_diff_until_2000-01-01T00:00:00.001Z",
+            "/commit_diff_until_2000-01-01T00:00:00.002Z",
+            "/commit_diff_until_2000-01-01T00:00:00.003Z",
+            "/commit_diff_until_2000-01-01T00:00:00.004Z");
+    PCollection<String> filteredFilenames =
+        pipeline
+            .apply(
+                "Generate All Filenames",
+                Create.of(commitLogFilenames).withCoder(StringUtf8Coder.of()))
+            .apply(
+                "Filtered by Time",
+                CommitLogTransforms.filterCommitLogsByTime(
+                    DateTime.parse("2000-01-01T00:00:00.001Z"),
+                    DateTime.parse("2000-01-01T00:00:00.003Z")));
+    PAssert.that(filteredFilenames)
+        .containsInAnyOrder(
+            "/commit_diff_until_2000-01-01T00:00:00.001Z",
+            "/commit_diff_until_2000-01-01T00:00:00.002Z");
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void loadOneCommitLogFile() {
+    PCollection<VersionedEntity> entities =
+        pipeline
+            .apply(
+                "Get CommitLog file patterns",
+                CommitLogTransforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath()))
+            .apply("Find CommitLogs", Transforms.getFilesByPatterns())
+            .apply(CommitLogTransforms.loadCommitLogsFromFiles());
+
+    PCollection<Long> timestamps =
+        entities.apply(
+            "Extract commitTimeMillis",
+            ParDo.of(
+                new DoFn<VersionedEntity, Long>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity entity, OutputReceiver<Long> out) {
+                    out.output(entity.commitTimeMills());
+                  }
+                }));
+    PAssert.that(timestamps)
+        .containsInAnyOrder(
+            fakeClock.nowUtc().getMillis() - 2,
+            fakeClock.nowUtc().getMillis() - 1,
+            fakeClock.nowUtc().getMillis() - 1);
+
+    PCollection<Entity> datastoreEntities =
+        entities.apply(
+            "To Datastore Entities",
+            ParDo.of(
+                new DoFn<VersionedEntity, Entity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity entity, OutputReceiver<Entity> out) {
+                    entity.getEntity().ifPresent(out::output);
+                  }
+                }));
+
+    PAssert.that(datastoreEntities).containsInAnyOrder(persistedEntities);
+
+    pipeline.run();
+  }
+}

core/src/test/java/google/registry/beam/initsql/ExportloadingTransformsTest.java

@@ -0,0 +1,201 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatastoreHelper.newContactResource;
+import static google.registry.testing.DatastoreHelper.newDomainBase;
+import static google.registry.testing.DatastoreHelper.newRegistry;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.Key;
+import google.registry.backup.VersionedEntity;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.Ofy;
+import google.registry.model.registry.Registry;
+import google.registry.testing.FakeClock;
+import google.registry.testing.InjectRule;
+import java.io.File;
+import java.io.Serializable;
+import java.util.Collections;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.DateTime;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Unit tests for {@link ExportLoadingTransforms}.
+ *
+ * <p>This class implements {@link Serializable} so that test {@link DoFn} classes may be inlined.
+ */
+// TODO(weiminyu): Upgrade to JUnit5 when TestPipeline is upgraded. It is also easy to adapt with
+// a wrapper.
+@RunWith(JUnit4.class)
+public class ExportloadingTransformsTest implements Serializable {
+  private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z");
+
+  private static final ImmutableList<Class<?>> ALL_KINDS =
+      ImmutableList.of(Registry.class, ContactResource.class, DomainBase.class);
+  private static final ImmutableList<String> ALL_KIND_STRS =
+      ALL_KINDS.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
+
+  @Rule public final transient TemporaryFolder exportRootDir = new TemporaryFolder();
+
+  @Rule public final transient InjectRule injectRule = new InjectRule();
+
+  @Rule
+  public final transient TestPipeline pipeline =
+      TestPipeline.create().enableAbandonedNodeEnforcement(true);
+
+  private FakeClock fakeClock;
+  private transient BackupTestStore store;
+  private File exportDir;
+  // Canned data that are persisted to Datastore, used by assertions in tests.
+  // TODO(weiminyu): use Ofy entity pojos directly.
+  private transient ImmutableList<Entity> persistedEntities;
+
+  @Before
+  public void beforeEach() throws Exception {
+    fakeClock = new FakeClock(START_TIME);
+    store = new BackupTestStore(fakeClock);
+    injectRule.setStaticField(Ofy.class, "clock", fakeClock);
+
+    Registry registry = newRegistry("tld1", "TLD1");
+    store.insertOrUpdate(registry);
+    ContactResource contact1 = newContactResource("contact_1");
+    DomainBase domain1 = newDomainBase("domain1.tld1", contact1);
+    store.insertOrUpdate(contact1, domain1);
+    persistedEntities =
+        ImmutableList.of(registry, contact1, domain1).stream()
+            .map(ofyEntity -> tm().transact(() -> ofy().save().toEntity(ofyEntity)))
+            .collect(ImmutableList.toImmutableList());
+
+    exportDir =
+        store.export(exportRootDir.getRoot().getAbsolutePath(), ALL_KINDS, Collections.EMPTY_SET);
+  }
+
+  @After
+  public void afterEach() throws Exception {
+    if (store != null) {
+      store.close();
+      store = null;
+    }
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void getExportFilePatterns() {
+    PCollection<String> patterns =
+        pipeline.apply(
+            "Get Datastore file patterns",
+            ExportLoadingTransforms.getDatastoreExportFilePatterns(
+                exportDir.getAbsolutePath(), ALL_KIND_STRS));
+
+    ImmutableList<String> expectedPatterns =
+        ImmutableList.of(
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/input-*",
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/input-*",
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_ContactResource/input-*");
+
+    PAssert.that(patterns).containsInAnyOrder(expectedPatterns);
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void getFilesByPatterns() {
+    PCollection<Metadata> fileMetas =
+        pipeline
+            .apply(
+                "File patterns to metadata",
+                Create.of(
+                        exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/input-*",
+                        exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/input-*",
+                        exportDir.getAbsolutePath()
+                            + "/all_namespaces/kind_ContactResource/input-*")
+                    .withCoder(StringUtf8Coder.of()))
+            .apply(Transforms.getFilesByPatterns());
+
+    // Transform fileMetas to file names for assertions.
+    PCollection<String> fileNames =
+        fileMetas.apply(
+            "File metadata to path string",
+            ParDo.of(
+                new DoFn<Metadata, String>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element Metadata metadata, OutputReceiver<String> out) {
+                    out.output(metadata.resourceId().toString());
+                  }
+                }));
+
+    ImmutableList<String> expectedFilenames =
+        ImmutableList.of(
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/input-0",
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/input-0",
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_ContactResource/input-0");
+
+    PAssert.that(fileNames).containsInAnyOrder(expectedFilenames);
+
+    pipeline.run();
+  }
+
+  @Test
+  public void loadDataFromFiles() {
+    PCollection<VersionedEntity> taggedRecords =
+        pipeline
+            .apply(
+                "Get Datastore file patterns",
+                ExportLoadingTransforms.getDatastoreExportFilePatterns(
+                    exportDir.getAbsolutePath(), ALL_KIND_STRS))
+            .apply("Find Datastore files", Transforms.getFilesByPatterns())
+            .apply("Load from Datastore files", ExportLoadingTransforms.loadExportDataFromFiles());
+
+    // Transform bytes to pojo for analysis
+    PCollection<Entity> entities =
+        taggedRecords.apply(
+            "Raw records to Entity",
+            ParDo.of(
+                new DoFn<VersionedEntity, Entity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity versionedEntity, OutputReceiver<Entity> out) {
+                    out.output(versionedEntity.getEntity().get());
+                  }
+                }));
+
+    PAssert.that(entities).containsInAnyOrder(persistedEntities);
+
+    pipeline.run();
+  }
+}

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -195,14 +195,10 @@ public class Spec11PipelineTest {
             new JSONObject()
                 .put("fullyQualifiedDomainName", "111.com")
                 .put("threatType", "MALWARE")
-                .put("threatEntryMetadata", "NONE")
-                .put("platformType", "WINDOWS")
                 .toString(),
             new JSONObject()
                 .put("fullyQualifiedDomainName", "222.com")
                 .put("threatType", "MALWARE")
-                .put("threatEntryMetadata", "NONE")
-                .put("platformType", "WINDOWS")
                 .toString());
   }
 

core/src/test/java/google/registry/dns/writer/clouddns/CloudDnsWriterTest.java

@@ -294,7 +294,7 @@ public class CloudDnsWriterTest {
 
     ImmutableSet.Builder<VKey<HostResource>> hostResourceRefBuilder = new ImmutableSet.Builder<>();
     for (HostResource nameserver : nameservers) {
-      hostResourceRefBuilder.add(nameserver.createKey());
+      hostResourceRefBuilder.add(nameserver.createVKey());
     }
 
     return newDomainBase(domainName)

core/src/test/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java

@@ -105,7 +105,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain =
         persistActiveDomain("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host1.createKey(), host2.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey(), host2.createVKey()))
             .build();
     persistResource(domain);
 
@@ -126,7 +126,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain1 =
         persistActiveDomain("example1.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host1.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey()))
             .build();
     persistResource(domain1);
 
@@ -134,7 +134,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain2 =
         persistActiveDomain("example2.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host2.createKey()))
+            .setNameservers(ImmutableSet.of(host2.createVKey()))
             .build();
     persistResource(domain2);
 
@@ -150,7 +150,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain1 =
         persistActiveDomain("example1.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host1.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey()))
             .build();
     persistResource(domain1);
 
@@ -158,7 +158,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain2 =
         persistActiveDomain("example2.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host2.createKey()))
+            .setNameservers(ImmutableSet.of(host2.createVKey()))
             .build();
     persistResource(domain2);
 
@@ -181,7 +181,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain =
         persistActiveDomain("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createVKey()))
             .setDsData(
                 ImmutableSet.of(
                     DelegationSignerData.create(1, 3, 1, base16().decode("0123456789ABCDEF"))))
@@ -206,7 +206,7 @@ public class DnsUpdateWriterTest {
         persistActiveDomain("example.tld")
             .asBuilder()
             .addStatusValue(StatusValue.SERVER_HOLD)
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createVKey()))
             .build();
     persistResource(domain);
 
@@ -250,7 +250,7 @@ public class DnsUpdateWriterTest {
         newDomainBase("example.tld")
             .asBuilder()
             .addSubordinateHost("ns1.example.tld")
-            .addNameserver(host.createKey())
+            .addNameserver(host.createVKey())
             .build());
 
     writer.publishHost("ns1.example.tld");
@@ -289,7 +289,7 @@ public class DnsUpdateWriterTest {
     persistResource(
         persistActiveDomain("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.com").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.com").createVKey()))
             .build());
 
     writer.publishHost("ns1.example.tld");
@@ -323,7 +323,8 @@ public class DnsUpdateWriterTest {
             .asBuilder()
             .addSubordinateHost("ns1.example.tld")
             .addNameservers(
-                ImmutableSet.of(externalNameserver.createKey(), inBailiwickNameserver.createKey()))
+                ImmutableSet.of(
+                    externalNameserver.createVKey(), inBailiwickNameserver.createVKey()))
             .build());
 
     writer.publishDomain("example.tld");
@@ -358,7 +359,7 @@ public class DnsUpdateWriterTest {
             .asBuilder()
             .addSubordinateHost("ns1.example.tld")
             .addSubordinateHost("foo.example.tld")
-            .addNameserver(inBailiwickNameserver.createKey())
+            .addNameserver(inBailiwickNameserver.createVKey())
             .build());
 
     writer.publishDomain("example.tld");
@@ -382,7 +383,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain =
         persistActiveDomain("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createVKey()))
             .build();
     persistResource(domain);
     when(mockResolver.send(any(Message.class))).thenReturn(messageWithResponseCode(Rcode.SERVFAIL));

core/src/test/java/google/registry/flows/EppLifecycleDomainTest.java

@@ -127,9 +127,7 @@ public class EppLifecycleDomainTest extends EppTestCase {
             ImmutableMap.of(
                 "DOMAIN", "example.tld",
                 "CRDATE", "2000-06-01T00:02:00Z",
-                // TODO(mcilwain): The exp. date should be restored back to 2002-06-01T00:02:00Z,
-                // but this is old behavior of being 1 year after the moment of the restore.
-                "EXDATE", "2001-07-01T00:03:00Z",
+                "EXDATE", "2002-06-01T00:02:00Z",
                 "UPDATE", "2000-07-01T00:03:00Z"));
 
     assertThatLogoutSucceeds();
@@ -203,11 +201,7 @@ public class EppLifecycleDomainTest extends EppTestCase {
             ImmutableMap.of(
                 "DOMAIN", "example.tld",
                 "CRDATE", "2000-06-01T00:02:00Z",
-                // TODO(mcilwain): The exp. date should be 2003-06-01T00:02:00Z, the same as its
-                // value prior to the deletion, because the year that was taken off when the
-                // autorenew was canceled will be re-added in renewal during the restore.
-                // For now though, the current behavior is 1 year after restore.
-                "EXDATE", "2003-07-05T00:03:00Z",
+                "EXDATE", "2003-06-01T00:02:00Z",
                 "UPDATE", "2002-07-05T00:03:00Z"));
 
     assertThatLogoutSucceeds();
@@ -289,10 +283,7 @@ public class EppLifecycleDomainTest extends EppTestCase {
             ImmutableMap.of(
                 "DOMAIN", "example.tld",
                 "CRDATE", "2000-06-01T00:02:00Z",
-                // TODO(mcilwain): The exp. date should be 2002-06-01T00:02:00Z, which is the
-                // current registration expiration time on the (deleted) domain, but for now is
-                // 1 year after restore.
-                "EXDATE", "2001-06-20T00:00:00Z",
+                "EXDATE", "2002-06-01T00:02:00Z",
                 "UPDATE", "2000-06-20T00:00:00Z"));
 
     assertThatLogoutSucceeds();

core/src/test/java/google/registry/flows/EppLifecycleHostTest.java

@@ -24,7 +24,6 @@ import static google.registry.testing.EppMetricSubject.assertThat;
 import static google.registry.testing.HostResourceSubject.assertAboutHosts;
 
 import com.google.common.collect.ImmutableMap;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.testing.AppEngineRule;
@@ -221,7 +220,7 @@ public class EppLifecycleHostTest extends EppTestCase {
         loadByForeignKey(DomainBase.class, "example.bar.foo.tld", timeAfterCreates).get();
     assertAboutHosts()
         .that(exampleBarFooTldHost)
-        .hasSuperordinateDomain(Key.create(exampleBarFooTldDomain));
+        .hasSuperordinateDomain(exampleBarFooTldDomain.createVKey());
     assertThat(exampleBarFooTldDomain.getSubordinateHosts())
         .containsExactly("ns1.example.bar.foo.tld");
 
@@ -231,14 +230,14 @@ public class EppLifecycleHostTest extends EppTestCase {
         loadByForeignKey(DomainBase.class, "example.foo.tld", timeAfterCreates).get();
     assertAboutHosts()
         .that(exampleFooTldHost)
-        .hasSuperordinateDomain(Key.create(exampleFooTldDomain));
+        .hasSuperordinateDomain(exampleFooTldDomain.createVKey());
     assertThat(exampleFooTldDomain.getSubordinateHosts()).containsExactly("ns1.example.foo.tld");
 
     HostResource exampleTldHost =
         loadByForeignKey(HostResource.class, "ns1.example.tld", timeAfterCreates).get();
     DomainBase exampleTldDomain =
         loadByForeignKey(DomainBase.class, "example.tld", timeAfterCreates).get();
-    assertAboutHosts().that(exampleTldHost).hasSuperordinateDomain(Key.create(exampleTldDomain));
+    assertAboutHosts().that(exampleTldHost).hasSuperordinateDomain(exampleTldDomain.createVKey());
     assertThat(exampleTldDomain.getSubordinateHosts()).containsExactly("ns1.example.tld");
 
     assertThatLogoutSucceeds();

core/src/test/java/google/registry/flows/contact/ContactTransferRequestFlowTest.java

@@ -52,6 +52,7 @@ import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import org.joda.time.DateTime;
 import org.junit.Before;
 import org.junit.Test;
@@ -129,9 +130,13 @@ public class ContactTransferRequestFlowTest
     // poll messages, the approval notice ones for gaining and losing registrars.
     assertPollMessagesEqual(
         Iterables.filter(
-            ofy().load()
+            ofy()
+                .load()
                 // Use toArray() to coerce the type to something keys() will accept.
-                .keys(contact.getTransferData().getServerApproveEntities().toArray(new Key<?>[]{}))
+                .keys(
+                    contact.getTransferData().getServerApproveEntities().stream()
+                        .map(VKey::getOfyKey)
+                        .toArray(Key[]::new))
                 .values(),
             PollMessage.class),
         ImmutableList.of(gainingApproveMessage, losingApproveMessage));

core/src/test/java/google/registry/flows/custom/TestDomainPricingCustomLogic.java

@@ -40,7 +40,7 @@ public class TestDomainPricingCustomLogic extends DomainPricingCustomLogic {
   public FeesAndCredits customizeRenewPrice(RenewPriceParameters priceParameters) {
     return priceParameters.domainName().toString().startsWith("costly-renew")
         ? addCustomFee(
-            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.RENEW))
+            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.RENEW, true))
         : priceParameters.feesAndCredits();
   }
 
@@ -48,7 +48,7 @@ public class TestDomainPricingCustomLogic extends DomainPricingCustomLogic {
   public FeesAndCredits customizeTransferPrice(TransferPriceParameters priceParameters) {
     return priceParameters.domainName().toString().startsWith("expensive")
         ? addCustomFee(
-            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.TRANSFER))
+            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.TRANSFER, true))
         : priceParameters.feesAndCredits();
   }
 
@@ -56,7 +56,7 @@ public class TestDomainPricingCustomLogic extends DomainPricingCustomLogic {
   public FeesAndCredits customizeUpdatePrice(UpdatePriceParameters priceParameters) {
     return priceParameters.domainName().toString().startsWith("non-free-update")
         ? addCustomFee(
-            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.UPDATE))
+            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.UPDATE, true))
         : priceParameters.feesAndCredits();
   }
 

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -93,6 +93,7 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.util.Map;
 import org.joda.money.Money;
@@ -668,13 +669,24 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
                 .setPendingTransferExpirationTime(clock.nowUtc())
                 .build());
     // The server-approve entities should all be deleted.
-    assertThat(ofy().load().key(oldTransferData.getServerApproveBillingEvent()).now()).isNull();
-    assertThat(ofy().load().key(oldTransferData.getServerApproveAutorenewEvent()).now()).isNull();
-    assertThat(ofy().load().key(oldTransferData.getServerApproveAutorenewPollMessage()).now())
+    assertThat(ofy().load().key(oldTransferData.getServerApproveBillingEvent().getOfyKey()).now())
+        .isNull();
+    assertThat(ofy().load().key(oldTransferData.getServerApproveAutorenewEvent().getOfyKey()).now())
+        .isNull();
+    assertThat(
+            ofy()
+                .load()
+                .key(oldTransferData.getServerApproveAutorenewPollMessage().getOfyKey())
+                .now())
         .isNull();
     assertThat(oldTransferData.getServerApproveEntities()).isNotEmpty(); // Just a sanity check.
     assertThat(
-            ofy().load().keys(oldTransferData.getServerApproveEntities().toArray(new Key<?>[] {})))
+            ofy()
+                .load()
+                .keys(
+                    oldTransferData.getServerApproveEntities().stream()
+                        .map(VKey::getOfyKey)
+                        .toArray(Key[]::new)))
         .isEmpty();
   }
 
@@ -693,7 +705,7 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
         loadByForeignKey(DomainBase.class, getUniqueIdFromCommand(), clock.nowUtc())
             .get()
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .build());
     // Persist another domain that's already been deleted and references this contact and host.
     persistResource(
@@ -703,7 +715,7 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
                 loadByForeignKey(ContactResource.class, "sh8013", clock.nowUtc())
                     .get()
                     .createVKey())
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .setDeletionTime(START_OF_TIME)
             .build());
     clock.advanceOneMilli();
@@ -719,7 +731,7 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
     persistResource(
         newHostResource("ns1." + getUniqueIdFromCommand())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(reloadResourceByForeignKey()))
+            .setSuperordinateDomain(reloadResourceByForeignKey().createVKey())
             .setDeletionTime(clock.nowUtc().minusDays(1))
             .build());
     clock.advanceOneMilli();
@@ -767,7 +779,7 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
         persistResource(
             newHostResource("ns1." + getUniqueIdFromCommand())
                 .asBuilder()
-                .setSuperordinateDomain(Key.create(reloadResourceByForeignKey()))
+                .setSuperordinateDomain(reloadResourceByForeignKey().createVKey())
                 .build());
     persistResource(
         domain.asBuilder().addSubordinateHost(subordinateHost.getFullyQualifiedHostName()).build());

core/src/test/java/google/registry/flows/domain/DomainInfoFlowTest.java

@@ -118,19 +118,19 @@ public class DomainInfoFlowTest extends ResourceFlowTestCase<DomainInfoFlow, Dom
                         DesignatedContact.create(Type.ADMIN, contact.createVKey()),
                         DesignatedContact.create(Type.TECH, contact.createVKey())))
                 .setNameservers(
-                    inactive ? null : ImmutableSet.of(host1.createKey(), host2.createKey()))
+                    inactive ? null : ImmutableSet.of(host1.createVKey(), host2.createVKey()))
                 .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("2fooBAR")))
                 .build());
     // Set the superordinate domain of ns1.example.com to example.com. In reality, this would have
     // happened in the flow that created it, but here we just overwrite it in Datastore.
-    host1 = persistResource(host1.asBuilder().setSuperordinateDomain(Key.create(domain)).build());
+    host1 = persistResource(host1.asBuilder().setSuperordinateDomain(domain.createVKey()).build());
     // Create a subordinate host that is not delegated to by anyone.
     host3 =
         persistResource(
             new HostResource.Builder()
                 .setFullyQualifiedHostName("ns2.example.tld")
                 .setRepoId("3FF-TLD")
-                .setSuperordinateDomain(Key.create(domain))
+                .setSuperordinateDomain(domain.createVKey())
                 .build());
     // Add the subordinate host keys to the existing domain.
     domain =
@@ -294,7 +294,7 @@ public class DomainInfoFlowTest extends ResourceFlowTestCase<DomainInfoFlow, Dom
                 ImmutableSet.of(
                     DelegationSignerData.create(
                         12345, 3, 1, base16().decode("49FD46E6C4B45C55D4AC"))))
-            .setNameservers(ImmutableSet.of(host1.createKey(), host3.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey(), host3.createVKey()))
             .build());
     doSuccessfulTest("domain_info_response_dsdata.xml", false);
   }

core/src/test/java/google/registry/flows/domain/DomainRestoreRequestFlowTest.java

@@ -72,6 +72,7 @@ import google.registry.model.reporting.DomainTransactionRecord.TransactionReport
 import google.registry.model.reporting.HistoryEntry;
 import java.util.Map;
 import org.joda.money.Money;
+import org.joda.time.DateTime;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -93,6 +94,10 @@ public class DomainRestoreRequestFlowTest
   }
 
   void persistPendingDeleteDomain() throws Exception {
+    persistPendingDeleteDomain(clock.nowUtc().plusYears(5).plusDays(45));
+  }
+
+  void persistPendingDeleteDomain(DateTime expirationTime) throws Exception {
     DomainBase domain = newDomainBase(getUniqueIdFromCommand());
     HistoryEntry historyEntry =
         persistResource(
@@ -103,7 +108,7 @@ public class DomainRestoreRequestFlowTest
     persistResource(
         domain
             .asBuilder()
-            .setRegistrationExpirationTime(clock.nowUtc().plusYears(5).plusDays(45))
+            .setRegistrationExpirationTime(expirationTime)
             .setDeletionTime(clock.nowUtc().plusDays(35))
             .addGracePeriod(
                 GracePeriod.create(
@@ -129,9 +134,10 @@ public class DomainRestoreRequestFlowTest
   }
 
   @Test
-  public void testSuccess() throws Exception {
+  public void testSuccess_expiryStillInFuture_notExtended() throws Exception {
     setEppInput("domain_update_restore_request.xml", ImmutableMap.of("DOMAIN", "example.tld"));
-    persistPendingDeleteDomain();
+    DateTime expirationTime = clock.nowUtc().plusYears(5).plusDays(45);
+    persistPendingDeleteDomain(expirationTime);
     assertTransactionalFlow(true);
     // Double check that we see a poll message in the future for when the delete happens.
     assertThat(getPollMessages("TheRegistrar", clock.nowUtc().plusMonths(1))).hasSize(1);
@@ -140,11 +146,79 @@ public class DomainRestoreRequestFlowTest
     HistoryEntry historyEntryDomainRestore =
         getOnlyHistoryEntryOfType(domain, HistoryEntry.Type.DOMAIN_RESTORE);
     assertThat(ofy().load().key(domain.getAutorenewBillingEvent()).now().getEventTime())
-        .isEqualTo(clock.nowUtc().plusYears(1));
+        .isEqualTo(expirationTime);
+    assertAboutDomains()
+        .that(domain)
+        // New expiration time should be the same as from before the deletion.
+        .hasRegistrationExpirationTime(expirationTime)
+        .and()
+        .doesNotHaveStatusValue(StatusValue.PENDING_DELETE)
+        .and()
+        .hasDeletionTime(END_OF_TIME)
+        .and()
+        .hasOneHistoryEntryEachOfTypes(
+            HistoryEntry.Type.DOMAIN_DELETE, HistoryEntry.Type.DOMAIN_RESTORE)
+        .and()
+        .hasLastEppUpdateTime(clock.nowUtc())
+        .and()
+        .hasLastEppUpdateClientId("TheRegistrar");
+    assertThat(domain.getGracePeriods()).isEmpty();
+    assertDnsTasksEnqueued("example.tld");
+    // The poll message for the delete should now be gone. The only poll message should be the new
+    // autorenew poll message.
+    assertPollMessages(
+        "TheRegistrar",
+        new PollMessage.Autorenew.Builder()
+            .setTargetId("example.tld")
+            .setClientId("TheRegistrar")
+            .setEventTime(domain.getRegistrationExpirationTime())
+            .setAutorenewEndTime(END_OF_TIME)
+            .setMsg("Domain was auto-renewed.")
+            .setParent(historyEntryDomainRestore)
+            .build());
+    // There should be a onetime for the restore and a new recurring billing event, but no renew
+    // onetime.
+    assertBillingEvents(
+        new BillingEvent.Recurring.Builder()
+            .setReason(Reason.RENEW)
+            .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
+            .setTargetId("example.tld")
+            .setClientId("TheRegistrar")
+            .setEventTime(expirationTime)
+            .setRecurrenceEndTime(END_OF_TIME)
+            .setParent(historyEntryDomainRestore)
+            .build(),
+        new BillingEvent.OneTime.Builder()
+            .setReason(Reason.RESTORE)
+            .setTargetId("example.tld")
+            .setClientId("TheRegistrar")
+            .setCost(Money.of(USD, 17))
+            .setPeriodYears(1)
+            .setEventTime(clock.nowUtc())
+            .setBillingTime(clock.nowUtc())
+            .setParent(historyEntryDomainRestore)
+            .build());
+  }
+
+  @Test
+  public void testSuccess_expiryInPast_extendedByOneYear() throws Exception {
+    setEppInput("domain_update_restore_request.xml", ImmutableMap.of("DOMAIN", "example.tld"));
+    DateTime expirationTime = clock.nowUtc().minusDays(20);
+    DateTime newExpirationTime = expirationTime.plusYears(1);
+    persistPendingDeleteDomain(expirationTime);
+    assertTransactionalFlow(true);
+    // Double check that we see a poll message in the future for when the delete happens.
+    assertThat(getPollMessages("TheRegistrar", clock.nowUtc().plusMonths(1))).hasSize(1);
+    runFlowAssertResponse(loadFile("generic_success_response.xml"));
+    DomainBase domain = reloadResourceByForeignKey();
+    HistoryEntry historyEntryDomainRestore =
+        getOnlyHistoryEntryOfType(domain, HistoryEntry.Type.DOMAIN_RESTORE);
+    assertThat(ofy().load().key(domain.getAutorenewBillingEvent()).now().getEventTime())
+        .isEqualTo(newExpirationTime);
     assertAboutDomains()
         .that(domain)
         // New expiration time should be exactly a year from now.
-        .hasRegistrationExpirationTime(clock.nowUtc().plusYears(1))
+        .hasRegistrationExpirationTime(newExpirationTime)
         .and()
         .doesNotHaveStatusValue(StatusValue.PENDING_DELETE)
         .and()
@@ -178,7 +252,7 @@ public class DomainRestoreRequestFlowTest
             .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
             .setTargetId("example.tld")
             .setClientId("TheRegistrar")
-            .setEventTime(domain.getRegistrationExpirationTime())
+            .setEventTime(newExpirationTime)
             .setRecurrenceEndTime(END_OF_TIME)
             .setParent(historyEntryDomainRestore)
             .build(),

core/src/test/java/google/registry/flows/domain/DomainTransferFlowTestCase.java

@@ -29,7 +29,6 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.flows.Flow;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.model.EppResource;
@@ -109,15 +108,16 @@ public class DomainTransferFlowTestCase<F extends Flow, R extends EppResource>
             clock.nowUtc(),
             DateTime.parse("1999-04-03T22:00:00.0Z"),
             REGISTRATION_EXPIRATION_TIME);
-    subordinateHost = persistResource(
-        new HostResource.Builder()
-            .setRepoId("2-".concat(Ascii.toUpperCase(tld)))
-            .setFullyQualifiedHostName("ns1." + label + "." + tld)
-            .setPersistedCurrentSponsorClientId("TheRegistrar")
-            .setCreationClientId("TheRegistrar")
-            .setCreationTimeForTest(DateTime.parse("1999-04-03T22:00:00.0Z"))
-            .setSuperordinateDomain(Key.create(domain))
-            .build());
+    subordinateHost =
+        persistResource(
+            new HostResource.Builder()
+                .setRepoId("2-".concat(Ascii.toUpperCase(tld)))
+                .setFullyQualifiedHostName("ns1." + label + "." + tld)
+                .setPersistedCurrentSponsorClientId("TheRegistrar")
+                .setCreationClientId("TheRegistrar")
+                .setCreationTimeForTest(DateTime.parse("1999-04-03T22:00:00.0Z"))
+                .setSuperordinateDomain(domain.createVKey())
+                .build());
     domain =
         persistResource(
             domain

core/src/test/java/google/registry/flows/domain/DomainTransferRequestFlowTest.java

@@ -27,6 +27,7 @@ import static google.registry.model.registry.Registry.TldState.QUIET_PERIOD;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.TRANSFER_SUCCESSFUL;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_CREATE;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_TRANSFER_REQUEST;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatastoreHelper.assertBillingEvents;
 import static google.registry.testing.DatastoreHelper.assertBillingEventsEqual;
 import static google.registry.testing.DatastoreHelper.assertPollMessagesEqual;
@@ -290,13 +291,13 @@ public class DomainTransferRequestFlowTest
     // Assert that the domain's TransferData server-approve billing events match the above.
     if (expectTransferBillingEvent) {
       assertBillingEventsEqual(
-          ofy().load().key(domain.getTransferData().getServerApproveBillingEvent()).now(),
+          tm().load(domain.getTransferData().getServerApproveBillingEvent()),
           optionalTransferBillingEvent.get());
     } else {
       assertThat(domain.getTransferData().getServerApproveBillingEvent()).isNull();
     }
     assertBillingEventsEqual(
-        ofy().load().key(domain.getTransferData().getServerApproveAutorenewEvent()).now(),
+        tm().load(domain.getTransferData().getServerApproveAutorenewEvent()),
         gainingClientAutorenew);
     // Assert that the full set of server-approve billing events is exactly the extra ones plus
     // the transfer billing event (if present) and the gaining client autorenew.
@@ -309,7 +310,10 @@ public class DomainTransferRequestFlowTest
             ofy()
                 .load()
                 // Use toArray() to coerce the type to something keys() will accept.
-                .keys(domain.getTransferData().getServerApproveEntities().toArray(new Key<?>[] {}))
+                .keys(
+                    domain.getTransferData().getServerApproveEntities().stream()
+                        .map(VKey::getOfyKey)
+                        .toArray(Key[]::new))
                 .values(),
             BillingEvent.class),
         Sets.union(expectedServeApproveBillingEvents, extraBillingEvents));
@@ -410,7 +414,7 @@ public class DomainTransferRequestFlowTest
 
     // Assert that the poll messages show up in the TransferData server approve entities.
     assertPollMessagesEqual(
-        ofy().load().key(domain.getTransferData().getServerApproveAutorenewPollMessage()).now(),
+        tm().load(domain.getTransferData().getServerApproveAutorenewPollMessage()),
         autorenewPollMessage);
     // Assert that the full set of server-approve poll messages is exactly the server approve
     // OneTime messages to gaining and losing registrars plus the gaining client autorenew.
@@ -419,7 +423,10 @@ public class DomainTransferRequestFlowTest
             ofy()
                 .load()
                 // Use toArray() to coerce the type to something keys() will accept.
-                .keys(domain.getTransferData().getServerApproveEntities().toArray(new Key<?>[] {}))
+                .keys(
+                    domain.getTransferData().getServerApproveEntities().stream()
+                        .map(VKey::getOfyKey)
+                        .toArray(Key[]::new))
                 .values(),
             PollMessage.class),
         ImmutableList.of(

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -139,7 +139,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
                         DesignatedContact.create(Type.ADMIN, mak21Contact.createVKey()),
                         DesignatedContact.create(Type.BILLING, mak21Contact.createVKey())))
                 .setRegistrant(mak21Contact.createVKey())
-                .setNameservers(ImmutableSet.of(host.createKey()))
+                .setNameservers(ImmutableSet.of(host.createVKey()))
                 .build());
     historyEntryDomainCreate =
         persistResource(
@@ -162,7 +162,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
                     ImmutableSet.of(
                         DesignatedContact.create(Type.TECH, sh8013Contact.createVKey()),
                         DesignatedContact.create(Type.ADMIN, unusedContact.createVKey())))
-                .setNameservers(ImmutableSet.of(host.createKey()))
+                .setNameservers(ImmutableSet.of(host.createVKey()))
                 .build());
     historyEntryDomainCreate =
         persistResource(
@@ -263,7 +263,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
             loadByForeignKey(
                     HostResource.class, String.format("ns%d.example.foo", i), clock.nowUtc())
                 .get()
-                .createKey());
+                .createVKey());
       }
     }
     persistResource(
@@ -290,7 +290,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
     for (int i = 0; i < 26; i++) {
       HostResource host = persistActiveHost(String.format("max_test_%d.example.tld", i));
       if (i < 13) {
-        nameservers.add(host.createKey());
+        nameservers.add(host.createVKey());
       }
     }
     ImmutableList.Builder<DesignatedContact> contactsBuilder = new ImmutableList.Builder<>();
@@ -378,19 +378,19 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
                 ImmutableSet.of(
                     loadByForeignKey(HostResource.class, "ns1.example.tld", clock.nowUtc())
                         .get()
-                        .createKey()))
+                        .createVKey()))
             .build());
     clock.advanceOneMilli();
     assertTransactionalFlow(true);
     runFlowAssertResponse(loadFile("generic_success_response.xml"));
     domain = reloadResourceByForeignKey();
-    assertThat(domain.getNameservers()).containsExactly(addedHost.createKey());
+    assertThat(domain.getNameservers()).containsExactly(addedHost.createVKey());
     assertThat(domain.getSubordinateHosts()).containsExactly("ns1.example.tld", "ns2.example.tld");
     HostResource existingHost =
         loadByForeignKey(HostResource.class, "ns1.example.tld", clock.nowUtc()).get();
     addedHost = loadByForeignKey(HostResource.class, "ns2.example.tld", clock.nowUtc()).get();
-    assertThat(existingHost.getSuperordinateDomain()).isEqualTo(Key.create(domain));
-    assertThat(addedHost.getSuperordinateDomain()).isEqualTo(Key.create(domain));
+    assertThat(existingHost.getSuperordinateDomain()).isEqualTo(domain.createVKey());
+    assertThat(addedHost.getSuperordinateDomain()).isEqualTo(domain.createVKey());
   }
 
   @Test
@@ -1061,7 +1061,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
                 ImmutableSet.of(
                     loadByForeignKey(HostResource.class, "ns1.example.foo", clock.nowUtc())
                         .get()
-                        .createKey()))
+                        .createVKey()))
             .build());
     EppException thrown = assertThrows(AddRemoveSameValueException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
@@ -1202,13 +1202,13 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
         .doesNotContain(
             loadByForeignKey(HostResource.class, "ns2.example.foo", clock.nowUtc())
                 .get()
-                .createKey());
+                .createVKey());
     runFlow();
     assertThat(reloadResourceByForeignKey().getNameservers())
         .contains(
             loadByForeignKey(HostResource.class, "ns2.example.foo", clock.nowUtc())
                 .get()
-                .createKey());
+                .createVKey());
   }
 
   @Test
@@ -1279,7 +1279,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
             .addNameserver(
                 loadByForeignKey(HostResource.class, "ns2.example.foo", clock.nowUtc())
                     .get()
-                    .createKey())
+                    .createVKey())
             .build());
     persistResource(
         Registry.get("tld")
@@ -1291,7 +1291,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
         .contains(
             loadByForeignKey(HostResource.class, "ns1.example.foo", clock.nowUtc())
                 .get()
-                .createKey());
+                .createVKey());
     clock.advanceOneMilli();
     runFlow();
     assertThat(reloadResourceByForeignKey().getNameservers())

core/src/test/java/google/registry/flows/host/HostCreateFlowTest.java

@@ -34,7 +34,6 @@ import static org.junit.Assert.assertThrows;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
 import google.registry.flows.FlowUtils.IpAddressVersionMismatchException;
 import google.registry.flows.ResourceFlowTestCase;
@@ -119,7 +118,7 @@ public class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, Hos
     HostResource host = reloadResourceByForeignKey();
     DomainBase superordinateDomain =
         loadByForeignKey(DomainBase.class, "example.tld", clock.nowUtc()).get();
-    assertAboutHosts().that(host).hasSuperordinateDomain(Key.create(superordinateDomain));
+    assertAboutHosts().that(host).hasSuperordinateDomain(superordinateDomain.createVKey());
     assertThat(superordinateDomain.getSubordinateHosts()).containsExactly("ns1.example.tld");
     assertDnsTasksEnqueued("ns1.example.tld");
   }
@@ -148,7 +147,7 @@ public class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, Hos
     HostResource host = reloadResourceByForeignKey();
     DomainBase superordinateDomain =
         loadByForeignKey(DomainBase.class, "example.tld", clock.nowUtc()).get();
-    assertAboutHosts().that(host).hasSuperordinateDomain(Key.create(superordinateDomain));
+    assertAboutHosts().that(host).hasSuperordinateDomain(superordinateDomain.createVKey());
     assertThat(superordinateDomain.getSubordinateHosts()).containsExactly("ns1.example.tld");
     assertDnsTasksEnqueued("ns1.example.tld");
   }