Refactor Fee handling so that each fee knows if it's premium (#626)

* Refactor Fee handling so that each fee knows if it's premium

This is a noop for now, as the new isPremium boolean isn't yet used by anything,
but it will be used in follow-up PRs to add additional fee information using the
fee extension (see: b/157621273).  Specifically what we're trying to do here is
return <fee:command name="create" standard="1"> (using the finalized version of
the fee extension) when an entire command has no premium fee associated with
it. And in the current earlier versions of the fee extension that we support,
we'll want to display the correct fee amount and class for creates/checks on
reserved domains when a valid allocation token is passed. This also needs the
isPremium information.

There are no testing implications yet because isPremium isn't exposed anywhere,
but there will definitely be lots of test changes once it's feeding into EPP-
visible changes.

* Rename things, add method Javadoc

* Apply formatting

README.md

@@ -33,7 +33,7 @@ running system:
 *   View the source code for the [GAE app](https://github.com/google/nomulus/tree/master/core/src/main/java/google/registry)
     and for the [GKE proxy](https://github.com/google/nomulus/tree/master/proxy/src/main/java/google/registry)
 *   [Other docs](https://github.com/google/nomulus/tree/master/docs)
-*   [Javadoc](https://nomulus.foo/javadoc/latest/)
+*   [Javadoc](https://javadoc.nomulus.foo/)
 *   [Nomulus discussion
     group](https://groups.google.com/forum/#!forum/nomulus-discuss), for any
     other questions

build.gradle

@@ -458,6 +458,7 @@ task javadoc(type: Javadoc) {
   source javadocSource
   classpath = files(javadocClasspath)
   destinationDir = file("${buildDir}/docs/javadoc")
+  options.encoding = "UTF-8"
   // In a lot of places we don't write @return so suppress warnings about that.
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
@@ -467,4 +468,4 @@ task javadoc(type: Javadoc) {
 
 tasks.build.dependsOn(tasks.javadoc)
 
-javadocDependentTasks.each { tasks.javadoc.shouldRunAfter(it) }
+javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }

core/src/main/java/google/registry/backup/AppEngineEnvironment.java

@@ -0,0 +1,66 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.apphosting.api.ApiProxy;
+import com.google.apphosting.api.ApiProxy.Environment;
+import com.google.common.collect.ImmutableMap;
+import java.io.Closeable;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+
+/**
+ * Sets up a placeholder {@link Environment} on a non-AppEngine platform so that Datastore Entities
+ * can be deserialized. See {@code DatastoreEntityExtension} in test source for more information.
+ */
+public class AppEngineEnvironment implements Closeable {
+
+  private static final Environment PLACEHOLDER_ENV = createAppEngineEnvironment();
+
+  private boolean isPlaceHolderNeeded;
+
+  AppEngineEnvironment() {
+    isPlaceHolderNeeded = ApiProxy.getCurrentEnvironment() == null;
+    // isPlaceHolderNeeded may be true when we are invoked in a test with AppEngineRule.
+    if (isPlaceHolderNeeded) {
+      ApiProxy.setEnvironmentForCurrentThread(PLACEHOLDER_ENV);
+    }
+  }
+
+  @Override
+  public void close() {
+    if (isPlaceHolderNeeded) {
+      ApiProxy.setEnvironmentForCurrentThread(null);
+    }
+  }
+
+  /** Returns a placeholder {@link Environment} that can return hardcoded AppId and Attributes. */
+  private static Environment createAppEngineEnvironment() {
+    return (Environment)
+        Proxy.newProxyInstance(
+            Environment.class.getClassLoader(),
+            new Class[] {Environment.class},
+            (Object proxy, Method method, Object[] args) -> {
+              switch (method.getName()) {
+                case "getAppId":
+                  return "PlaceholderAppId";
+                case "getAttributes":
+                  return ImmutableMap.<String, Object>of();
+                default:
+                  throw new UnsupportedOperationException(method.getName());
+              }
+            });
+  }
+}

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -0,0 +1,95 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.backup.BackupUtils.createDeserializingIterator;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Streams;
+import google.registry.model.ImmutableObject;
+import google.registry.model.ofy.CommitLogCheckpoint;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Iterator;
+import java.util.stream.Stream;
+
+/**
+ * Helpers for reading CommitLog records from a file.
+ *
+ * <p>This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial
+ * population of the Cloud SQL database.
+ */
+public final class CommitLogImports {
+
+  private CommitLogImports() {}
+
+  /**
+   * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link
+   * ImmutableList} of {@link VersionedEntity} records. Upon completion the {@code inputStream} is
+   * closed.
+   *
+   * <p>The returned list may be empty, since CommitLogs are written at fixed intervals regardless
+   * if actual changes exist.
+   *
+   * <p>A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated)
+   * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence
+   * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
+   * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
+   */
+  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
+        InputStream input = new BufferedInputStream(inputStream)) {
+      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
+      checkState(commitLogs.hasNext());
+      checkState(commitLogs.next() instanceof CommitLogCheckpoint);
+
+      return Streams.stream(commitLogs)
+          .map(
+              e ->
+                  e instanceof CommitLogManifest
+                      ? VersionedEntity.fromManifest((CommitLogManifest) e)
+                      : Stream.of(VersionedEntity.fromMutation((CommitLogMutation) e)))
+          .flatMap(s -> s)
+          .collect(ImmutableList.toImmutableList());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /** Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link File}. */
+  public static ImmutableList<VersionedEntity> loadEntities(File commitLogFile) {
+    try {
+      return loadEntities(new FileInputStream(commitLogFile));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link
+   * ReadableByteChannel}.
+   */
+  public static ImmutableList<VersionedEntity> loadEntities(ReadableByteChannel channel) {
+    return loadEntities(Channels.newInputStream(channel));
+  }
+}

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -0,0 +1,160 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.appengine.api.datastore.Key;
+import com.google.auto.value.AutoValue;
+import com.google.auto.value.extension.memoized.Memoized;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+
+/**
+ * A Datastore {@link Entity Entity's} timestamped state.
+ *
+ * <p>For a new or updated Entity, its ProtocolBuffer bytes are stored along with its {@link Key}.
+ * For a deleted entity, only its {@link Key} is stored, and the {@link #entityProtoBytes} is left
+ * as null.
+ *
+ * <p>Note that {@link Optional java.util.Optional} is not serializable, therefore cannot be used as
+ * property type in this class.
+ */
+@AutoValue
+public abstract class VersionedEntity implements Serializable {
+
+  private static final long serialVersionUID = 1L;
+
+  public abstract long commitTimeMills();
+
+  /** The {@link Key} of the {@link Entity}. */
+  public abstract Key key();
+
+  /** Serialized form of the {@link Entity}. This property is {@code null} for a deleted Entity. */
+  @Nullable
+  abstract ImmutableBytes entityProtoBytes();
+
+  @Memoized
+  public Optional<Entity> getEntity() {
+    return Optional.ofNullable(entityProtoBytes())
+        .map(ImmutableBytes::getBytes)
+        .map(EntityTranslator::createFromPbBytes);
+  }
+
+  public boolean isDelete() {
+    return entityProtoBytes() == null;
+  }
+
+  /**
+   * Converts deleted entity keys in {@code manifest} into a {@link Stream} of {@link
+   * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
+   * information.
+   */
+  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+    long commitTimeMillis = manifest.getCommitTime().getMillis();
+    return manifest.getDeletions().stream()
+        .map(com.googlecode.objectify.Key::getRaw)
+        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+  }
+
+  /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
+  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+    return from(
+        com.googlecode.objectify.Key.create(mutation).getParent().getId(),
+        mutation.getEntityProtoBytes());
+  }
+
+  public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
+    return builder()
+        .entityProtoBytes(entityProtoBytes)
+        .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
+        .commitTimeMills(commitTimeMillis)
+        .build();
+  }
+
+  static Builder builder() {
+    return new AutoValue_VersionedEntity.Builder();
+  }
+
+  @AutoValue.Builder
+  public abstract static class Builder {
+
+    public abstract Builder commitTimeMills(long commitTimeMillis);
+
+    abstract Builder entityProtoBytes(ImmutableBytes bytes);
+
+    public abstract Builder key(Key key);
+
+    public abstract VersionedEntity build();
+
+    public Builder entityProtoBytes(byte[] bytes) {
+      return entityProtoBytes(new ImmutableBytes(bytes));
+    }
+  }
+
+  /**
+   * Wraps a byte array and prevents it from being modified by its original owner.
+   *
+   * <p>While this class seems an overkill, it exists for two reasons:
+   *
+   * <ul>
+   *   <li>It is easier to override the {@link #equals} method here (for value-equivalence check)
+   *       than to override the AutoValue-generated {@code equals} method.
+   *   <li>To appease the style checker, which forbids arrays as AutoValue property.
+   * </ul>
+   */
+  static final class ImmutableBytes implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private final byte[] bytes;
+
+    ImmutableBytes(byte[] bytes) {
+      this.bytes = Arrays.copyOf(bytes, bytes.length);
+    }
+
+    /**
+     * Returns the saved byte array. Invocation is restricted to trusted callers, who must not
+     * modify the array.
+     */
+    byte[] getBytes() {
+      return bytes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof ImmutableBytes)) {
+        return false;
+      }
+      ImmutableBytes that = (ImmutableBytes) o;
+      // Do not use Objects.equals, which checks reference identity instead of data in array.
+      return Arrays.equals(bytes, that.bytes);
+    }
+
+    @Override
+    public int hashCode() {
+      // Do not use Objects.hashCode, which hashes the reference, not the data in array.
+      return Arrays.hashCode(bytes);
+    }
+  }
+}

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -347,7 +347,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
       String resourceClientId = resource.getPersistedCurrentSponsorClientId();
       if (resource instanceof HostResource && ((HostResource) resource).isSubordinate()) {
         resourceClientId =
-            ofy().load().key(((HostResource) resource).getSuperordinateDomain()).now()
+            tm().load(((HostResource) resource).getSuperordinateDomain())
                 .cloneProjectedAtTime(now)
                 .getCurrentSponsorClientId();
       }
@@ -466,10 +466,11 @@ public class DeleteContactsAndHostsAction implements Runnable {
         HostResource host = (HostResource) existingResource;
         if (host.isSubordinate()) {
           dnsQueue.addHostRefreshTask(host.getFullyQualifiedHostName());
-          ofy().save().entity(
-              ofy().load().key(host.getSuperordinateDomain()).now().asBuilder()
-                  .removeSubordinateHost(host.getFullyQualifiedHostName())
-                  .build());
+          tm().saveNewOrUpdate(
+                  tm().load(host.getSuperordinateDomain())
+                      .asBuilder()
+                      .removeSubordinateHost(host.getFullyQualifiedHostName())
+                      .build());
         }
       } else {
         throw new IllegalStateException(

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -17,8 +17,7 @@ package google.registry.beam.initsql;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Strings.isNullOrEmpty;
 
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import org.joda.time.DateTime;
 
 /**
  * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a
@@ -30,13 +29,9 @@ public final class BackupPaths {
 
   private static final String WILDCARD_CHAR = "*";
   private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/input-%s";
-  /**
-   * Regex pattern that captures the kind string in a file name. Datastore places no restrictions on
-   * what characters may be used in a kind string.
-   */
-  private static final String FILENAME_TO_KIND_REGEX = ".+/all_namespaces/kind_(.+)/input-.+";
 
-  private static final Pattern FILENAME_TO_KIND_PATTERN = Pattern.compile(FILENAME_TO_KIND_REGEX);
+  public static final String COMMIT_LOG_NAME_PREFIX = "commit_diff_until_";
+  private static final String COMMIT_LOG_PATTERN_TEMPLATE = "%s/" + COMMIT_LOG_NAME_PREFIX + "*";
 
   /**
    * Returns a regex pattern that matches all Datastore export files of a given {@code kind}.
@@ -65,22 +60,15 @@ public final class BackupPaths {
     return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, Integer.toString(shard));
   }
 
-  /**
-   * Returns the 'kind' of entity stored in a file based on the file name.
-   *
-   * <p>This method poses low risk and greatly simplifies the implementation of some transforms in
-   * {@link ExportLoadingTransforms}.
-   *
-   * @see ExportLoadingTransforms
-   */
-  public static String getKindFromFileName(String fileName) {
+  public static String getCommitLogFileNamePattern(String commitLogDir) {
+    return String.format(COMMIT_LOG_PATTERN_TEMPLATE, commitLogDir);
+  }
+
+  /** Gets the Commit timestamp from a CommitLog file name. */
+  public static DateTime getCommitLogTimestamp(String fileName) {
     checkArgument(!isNullOrEmpty(fileName), "Null or empty fileName.");
-    Matcher matcher = FILENAME_TO_KIND_PATTERN.matcher(fileName);
-    checkArgument(
-        matcher.matches(),
-        "Illegal file name %s, should match %s.",
-        fileName,
-        FILENAME_TO_KIND_REGEX);
-    return matcher.group(1);
+    int start = fileName.lastIndexOf(COMMIT_LOG_NAME_PREFIX);
+    checkArgument(start >= 0, "Illegal file name %s.", fileName);
+    return DateTime.parse(fileName.substring(start + COMMIT_LOG_NAME_PREFIX.length()));
   }
 }

core/src/main/java/google/registry/beam/initsql/CommitLogTransforms.java

@@ -0,0 +1,109 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static google.registry.beam.initsql.BackupPaths.getCommitLogFileNamePattern;
+import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
+import static google.registry.beam.initsql.Transforms.processFiles;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+
+import google.registry.backup.CommitLogImports;
+import google.registry.backup.VersionedEntity;
+import java.io.IOException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.DateTime;
+
+/**
+ * {@link org.apache.beam.sdk.transforms.PTransform Pipeline transforms} for loading from Nomulus
+ * CommitLog files. They are all part of a transformation that loads raw records from a sequence of
+ * Datastore CommitLog files, and are broken apart for testing.
+ */
+public class CommitLogTransforms {
+
+  /**
+   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
+   * all Datastore CommitLog files.
+   */
+  public static PTransform<PBegin, PCollection<String>> getCommitLogFilePatterns(
+      String commitLogDir) {
+    return Create.of(getCommitLogFileNamePattern(commitLogDir)).withCoder(StringUtf8Coder.of());
+  }
+
+  /**
+   * Returns files with timestamps between {@code fromTime} (inclusive) and {@code endTime}
+   * (exclusive).
+   */
+  public static PTransform<PCollection<? extends String>, PCollection<String>>
+      filterCommitLogsByTime(DateTime fromTime, DateTime toTime) {
+    checkNotNull(fromTime, "fromTime");
+    checkNotNull(toTime, "toTime");
+    checkArgument(
+        fromTime.isBefore(toTime),
+        "Invalid time range: fromTime (%s) is before endTime (%s)",
+        fromTime,
+        toTime);
+    return ParDo.of(new FilterCommitLogFileByTime(fromTime, toTime));
+  }
+
+  /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
+  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
+      loadCommitLogsFromFiles() {
+    return processFiles(new LoadOneCommitLogsFile());
+  }
+
+  static class FilterCommitLogFileByTime extends DoFn<String, String> {
+    private final DateTime fromTime;
+    private final DateTime toTime;
+
+    public FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
+      this.fromTime = fromTime;
+      this.toTime = toTime;
+    }
+
+    @ProcessElement
+    public void processElement(@Element String fileName, OutputReceiver<String> out) {
+      DateTime timestamp = getCommitLogTimestamp(fileName);
+      if (isBeforeOrAt(fromTime, timestamp) && timestamp.isBefore(toTime)) {
+        out.output(fileName);
+      }
+    }
+  }
+
+  /**
+   * Reads a CommitLog file and converts its content into {@link VersionedEntity VersionedEntities}.
+   */
+  static class LoadOneCommitLogsFile extends DoFn<ReadableFile, VersionedEntity> {
+
+    @ProcessElement
+    public void processElement(@Element ReadableFile file, OutputReceiver<VersionedEntity> out) {
+      try {
+        CommitLogImports.loadEntities(file.open()).forEach(out::output);
+      } catch (IOException e) {
+        // Let the pipeline retry the whole file.
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

core/src/main/java/google/registry/beam/initsql/ExportLoadingTransforms.java

@@ -15,38 +15,26 @@
 package google.registry.beam.initsql;
 
 import static google.registry.beam.initsql.BackupPaths.getExportFileNamePattern;
-import static google.registry.beam.initsql.BackupPaths.getKindFromFileName;
-import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
-import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+import static google.registry.beam.initsql.Transforms.processFiles;
 
 import com.google.common.collect.ImmutableList;
+import google.registry.backup.VersionedEntity;
 import google.registry.tools.LevelDbLogReader;
 import java.io.IOException;
 import java.util.Collection;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.Compression;
-import org.apache.beam.sdk.io.FileIO;
 import org.apache.beam.sdk.io.FileIO.ReadableFile;
-import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
 import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.TypeDescriptor;
 
 /**
  * {@link PTransform Pipeline transforms} for loading from Datastore export files. They are all part
  * of a transformation that loads raw records from a Datastore export, and are broken apart for
  * testing.
- *
- * <p>We drop the 'kind' information in {@link #getDatastoreExportFilePatterns} and recover it later
- * using the file paths. Although we could have kept it by passing around {@link KV key-value
- * pairs}, the code would be more complicated, especially in {@link #loadDataFromFiles()}.
  */
 public class ExportLoadingTransforms {
 
@@ -63,51 +51,28 @@ public class ExportLoadingTransforms {
         .withCoder(StringUtf8Coder.of());
   }
 
-  /**
-   * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}.
-   */
-  public static PTransform<PCollection<String>, PCollection<Metadata>> getFilesByPatterns() {
-    return new PTransform<PCollection<String>, PCollection<Metadata>>() {
-      @Override
-      public PCollection<Metadata> expand(PCollection<String> input) {
-        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
-      }
-    };
+  /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */
+  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>
+      loadExportDataFromFiles() {
+    return processFiles(new LoadOneExportShard());
   }
 
   /**
-   * Returns a {@link PTransform} from file {@link Metadata} to {@link KV Key-Value pairs} with
-   * entity 'kind' as key and raw record as value.
-   */
-  public static PTransform<PCollection<Metadata>, PCollection<KV<String, byte[]>>>
-      loadDataFromFiles() {
-    return new PTransform<PCollection<Metadata>, PCollection<KV<String, byte[]>>>() {
-      @Override
-      public PCollection<KV<String, byte[]>> expand(PCollection<Metadata> input) {
-        return input
-            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
-            .apply(
-                MapElements.into(kvs(strings(), TypeDescriptor.of(ReadableFile.class)))
-                    .via(file -> KV.of(getKindFromFileName(file.getMetadata().toString()), file)))
-            .apply("Load One LevelDb File", ParDo.of(new LoadOneFile()));
-      }
-    };
-  }
-
-  /**
-   * Reads a LevelDb file and converts each raw record into a {@link KV pair} of kind and bytes.
+   * Reads a LevelDb file and converts each raw record into a {@link VersionedEntity}. All such
+   * entities use {@link Long#MIN_VALUE} as timestamp, so that they go before data from CommitLogs.
    *
    * <p>LevelDb files are not seekable because a large object may span multiple blocks. If a
    * sequential read fails, the file needs to be retried from the beginning.
    */
-  private static class LoadOneFile extends DoFn<KV<String, ReadableFile>, KV<String, byte[]>> {
+  private static class LoadOneExportShard extends DoFn<ReadableFile, VersionedEntity> {
+
+    private static final long TIMESTAMP = Long.MIN_VALUE;
 
     @ProcessElement
-    public void processElement(
-        @Element KV<String, ReadableFile> kv, OutputReceiver<KV<String, byte[]>> output) {
+    public void processElement(@Element ReadableFile file, OutputReceiver<VersionedEntity> output) {
       try {
-        LevelDbLogReader.from(kv.getValue().open())
-            .forEachRemaining(record -> output.output(KV.of(kv.getKey(), record)));
+        LevelDbLogReader.from(file.open())
+            .forEachRemaining(record -> output.output(VersionedEntity.from(TIMESTAMP, record)));
       } catch (IOException e) {
         // Let the pipeline retry the whole file.
         throw new RuntimeException(e);

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -0,0 +1,61 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import google.registry.backup.VersionedEntity;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * Common {@link PTransform pipeline transforms} used in pipelines that load from both Datastore
+ * export files and Nomulus CommitLog files.
+ */
+public class Transforms {
+
+  /**
+   * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}.
+   */
+  public static PTransform<PCollection<String>, PCollection<Metadata>> getFilesByPatterns() {
+    return new PTransform<PCollection<String>, PCollection<Metadata>>() {
+      @Override
+      public PCollection<Metadata> expand(PCollection<String> input) {
+        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+      }
+    };
+  }
+
+  /**
+   * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using
+   * caller-provided {@code transformer}.
+   */
+  public static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
+      DoFn<ReadableFile, VersionedEntity> transformer) {
+    return new PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>() {
+      @Override
+      public PCollection<VersionedEntity> expand(PCollection<Metadata> input) {
+        return input
+            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+            .apply(transformer.getClass().getSimpleName(), ParDo.of(transformer));
+      }
+    };
+  }
+}

core/src/main/java/google/registry/beam/spec11/ThreatMatch.java

@@ -24,22 +24,10 @@ import org.json.JSONObject;
 public abstract class ThreatMatch implements Serializable {
 
   private static final String THREAT_TYPE_FIELD = "threatType";
-  private static final String PLATFORM_TYPE_FIELD = "platformType";
-  private static final String METADATA_FIELD = "threatEntryMetadata";
   private static final String DOMAIN_NAME_FIELD = "fullyQualifiedDomainName";
 
   /** Returns what kind of threat it is (malware, phishing etc.) */
   public abstract String threatType();
-  /** Returns what platforms it affects (Windows, Linux etc.) */
-  abstract String platformType();
-  /**
-   * Returns a String representing a JSON Object containing arbitrary metadata associated with this
-   * threat, or "NONE" if there is no metadata to retrieve.
-   *
-   * <p>This ideally would be a {@link JSONObject} type, but can't be due to serialization
-   * requirements.
-   */
-  abstract String metadata();
   /** Returns the fully qualified domain name [SLD].[TLD] of the matched threat. */
   public abstract String fullyQualifiedDomainName();
 
@@ -52,29 +40,19 @@ public abstract class ThreatMatch implements Serializable {
   static ThreatMatch create(JSONObject threatMatchJSON, String fullyQualifiedDomainName)
       throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatchJSON.getString(THREAT_TYPE_FIELD),
-        threatMatchJSON.getString(PLATFORM_TYPE_FIELD),
-        threatMatchJSON.has(METADATA_FIELD)
-            ? threatMatchJSON.getJSONObject(METADATA_FIELD).toString()
-            : "NONE",
-        fullyQualifiedDomainName);
+        threatMatchJSON.getString(THREAT_TYPE_FIELD), fullyQualifiedDomainName);
   }
 
   /** Returns a {@link JSONObject} representing a subset of this object's data. */
   JSONObject toJSON() throws JSONException {
     return new JSONObject()
         .put(THREAT_TYPE_FIELD, threatType())
-        .put(PLATFORM_TYPE_FIELD, platformType())
-        .put(METADATA_FIELD, metadata())
         .put(DOMAIN_NAME_FIELD, fullyQualifiedDomainName());
   }
 
   /** Parses a {@link JSONObject} and returns an equivalent {@link ThreatMatch}. */
   public static ThreatMatch fromJSON(JSONObject threatMatch) throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatch.getString(THREAT_TYPE_FIELD),
-        threatMatch.getString(PLATFORM_TYPE_FIELD),
-        threatMatch.getString(METADATA_FIELD),
-        threatMatch.getString(DOMAIN_NAME_FIELD));
+        threatMatch.getString(THREAT_TYPE_FIELD), threatMatch.getString(DOMAIN_NAME_FIELD));
   }
 }

core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java

@@ -97,7 +97,7 @@ public final class ContactTransferApproveFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, gainingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java

@@ -93,7 +93,7 @@ public final class ContactTransferCancelFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, losingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java

@@ -90,7 +90,7 @@ public final class ContactTransferRejectFlow implements TransactionalFlow {
     ofy().save().<Object>entities(newContact, historyEntry, gainingPollMessage);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingContact.getTransferData().getServerApproveEntities());
+    tm().delete(existingContact.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newContact.getTransferData()))
         .build();

core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java

@@ -126,13 +126,15 @@ public final class ContactTransferRequestFlow implements TransactionalFlow {
     // If the transfer is server approved, this message will be sent to the gaining registrar. */
     PollMessage serverApproveGainingPollMessage =
         createGainingTransferPollMessage(targetId, serverApproveTransferData, historyEntry);
-    TransferData pendingTransferData = serverApproveTransferData.asBuilder()
-        .setTransferStatus(TransferStatus.PENDING)
-        .setServerApproveEntities(
-            ImmutableSet.of(
-                Key.create(serverApproveGainingPollMessage),
-                Key.create(serverApproveLosingPollMessage)))
-        .build();
+    TransferData pendingTransferData =
+        serverApproveTransferData
+            .asBuilder()
+            .setTransferStatus(TransferStatus.PENDING)
+            .setServerApproveEntities(
+                ImmutableSet.of(
+                    serverApproveGainingPollMessage.createVKey(),
+                    serverApproveLosingPollMessage.createVKey()))
+            .build();
     // When a transfer is requested, a poll message is created to notify the losing registrar.
     PollMessage requestPollMessage =
         createLosingTransferPollMessage(targetId, pendingTransferData, historyEntry).asBuilder()

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -690,7 +690,7 @@ public class DomainFlowUtils {
     List<Fee> fees = feeCommand.get().getFees();
     // The schema guarantees that at least one fee will be present.
     checkState(!fees.isEmpty());
-    BigDecimal total = BigDecimal.ZERO;
+    BigDecimal total = zeroInCurrency(feeCommand.get().getCurrency());
     for (Fee fee : fees) {
       if (!fee.hasDefaultAttributes()) {
         throw new UnsupportedFeeAttributeException();
@@ -938,6 +938,16 @@ public class DomainFlowUtils {
     }
   }
 
+  /**
+   * Returns zero for a specific currency.
+   *
+   * <p>{@link BigDecimal} has a concept of significant figures, so zero is not always zero. E.g.
+   * zero in USD is 0.00, whereas zero in Yen is 0, and zero in Dinars is 0.000 (!).
+   */
+  static BigDecimal zeroInCurrency(CurrencyUnit currencyUnit) {
+    return Money.of(currencyUnit, BigDecimal.ZERO).getAmount();
+  }
+
   /**
    * Check that if there's a claims notice it's on the claims list, and that if there's not one it's
    * not on the claims list.

core/src/main/java/google/registry/flows/domain/DomainPricingLogic.java

@@ -14,8 +14,9 @@
 
 package google.registry.flows.domain;
 
+import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
 import static google.registry.pricing.PricingEngineProxy.getDomainFeeClass;
-import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
+import static google.registry.pricing.PricingEngineProxy.getPricesForDomainName;
 
 import com.google.common.net.InternetDomainName;
 import google.registry.flows.EppException;
@@ -33,7 +34,6 @@ import google.registry.model.domain.fee.Fee;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.pricing.PremiumPricingEngine.DomainPrices;
 import google.registry.model.registry.Registry;
-import google.registry.pricing.PricingEngineProxy;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.util.Optional;
@@ -64,21 +64,27 @@ public final class DomainPricingLogic {
   public FeesAndCredits getCreatePrice(
       Registry registry,
       String domainName,
-      DateTime date,
+      DateTime dateTime,
       int years,
       boolean isAnchorTenant,
       Optional<AllocationToken> allocationToken)
       throws EppException {
     CurrencyUnit currency = registry.getCurrency();
+
+    BaseFee createFeeOrCredit;
     // Domain create cost is always zero for anchor tenants
-    Money domainCreateCost =
-        isAnchorTenant
-            ? Money.of(currency, BigDecimal.ZERO)
-            : getDomainCreateCostWithDiscount(domainName, date, years, allocationToken);
-    BaseFee createFeeOrCredit = Fee.create(domainCreateCost.getAmount(), FeeType.CREATE);
+    if (isAnchorTenant) {
+      createFeeOrCredit = Fee.create(zeroInCurrency(currency), FeeType.CREATE, false);
+    } else {
+      DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
+      Money domainCreateCost =
+          getDomainCreateCostWithDiscount(domainPrices, years, allocationToken);
+      createFeeOrCredit =
+          Fee.create(domainCreateCost.getAmount(), FeeType.CREATE, domainPrices.isPremium());
+    }
 
     // Create fees for the cost and the EAP fee, if any.
-    Fee eapFee = registry.getEapFeeFor(date);
+    Fee eapFee = registry.getEapFeeFor(dateTime);
     FeesAndCredits.Builder feesBuilder =
         new FeesAndCredits.Builder().setCurrency(currency).addFeeOrCredit(createFeeOrCredit);
     // Don't charge anchor tenants EAP fees.
@@ -92,7 +98,7 @@ public final class DomainPricingLogic {
             .setFeesAndCredits(feesBuilder.build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .setYears(years)
             .build());
   }
@@ -100,69 +106,73 @@ public final class DomainPricingLogic {
   /** Returns a new renew price for the pricer. */
   @SuppressWarnings("unused")
   public FeesAndCredits getRenewPrice(
-      Registry registry,
-      String domainName,
-      DateTime date,
-      int years)
-      throws EppException {
-    Money renewCost = getDomainRenewCost(domainName, date, years);
+      Registry registry, String domainName, DateTime dateTime, int years) throws EppException {
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
+    BigDecimal renewCost = domainPrices.getRenewCost().multipliedBy(years).getAmount();
     return customLogic.customizeRenewPrice(
         RenewPriceParameters.newBuilder()
             .setFeesAndCredits(
                 new FeesAndCredits.Builder()
                     .setCurrency(registry.getCurrency())
-                    .addFeeOrCredit(Fee.create(renewCost.getAmount(), FeeType.RENEW))
+                    .addFeeOrCredit(Fee.create(renewCost, FeeType.RENEW, domainPrices.isPremium()))
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .setYears(years)
             .build());
   }
 
   /** Returns a new restore price for the pricer. */
-  public FeesAndCredits getRestorePrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getRestorePrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
     FeesAndCredits feesAndCredits =
         new FeesAndCredits.Builder()
             .setCurrency(registry.getCurrency())
             .addFeeOrCredit(
-                Fee.create(getDomainRenewCost(domainName, date, 1).getAmount(), FeeType.RENEW))
+                Fee.create(
+                    domainPrices.getRenewCost().getAmount(),
+                    FeeType.RENEW,
+                    domainPrices.isPremium()))
             .addFeeOrCredit(
-                Fee.create(registry.getStandardRestoreCost().getAmount(), FeeType.RESTORE))
+                Fee.create(registry.getStandardRestoreCost().getAmount(), FeeType.RESTORE, false))
             .build();
     return customLogic.customizeRestorePrice(
         RestorePriceParameters.newBuilder()
             .setFeesAndCredits(feesAndCredits)
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns a new transfer price for the pricer. */
-  public FeesAndCredits getTransferPrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getTransferPrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
-    Money renewCost = getDomainRenewCost(domainName, date, 1);
+    DomainPrices domainPrices = getPricesForDomainName(domainName, dateTime);
     return customLogic.customizeTransferPrice(
         TransferPriceParameters.newBuilder()
             .setFeesAndCredits(
                 new FeesAndCredits.Builder()
                     .setCurrency(registry.getCurrency())
-                    .addFeeOrCredit(Fee.create(renewCost.getAmount(), FeeType.RENEW))
+                    .addFeeOrCredit(
+                        Fee.create(
+                            domainPrices.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            domainPrices.isPremium()))
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns a new update price for the pricer. */
-  public FeesAndCredits getUpdatePrice(Registry registry, String domainName, DateTime date)
+  public FeesAndCredits getUpdatePrice(Registry registry, String domainName, DateTime dateTime)
       throws EppException {
     CurrencyUnit currency = registry.getCurrency();
-    BaseFee feeOrCredit =
-        Fee.create(Money.zero(registry.getCurrency()).getAmount(), FeeType.UPDATE);
+    BaseFee feeOrCredit = Fee.create(zeroInCurrency(currency), FeeType.UPDATE, false);
     return customLogic.customizeUpdatePrice(
         UpdatePriceParameters.newBuilder()
             .setFeesAndCredits(
@@ -172,19 +182,19 @@ public final class DomainPricingLogic {
                     .build())
             .setRegistry(registry)
             .setDomainName(InternetDomainName.from(domainName))
-            .setAsOfDate(date)
+            .setAsOfDate(dateTime)
             .build());
   }
 
   /** Returns the fee class for a given domain and date. */
-  public Optional<String> getFeeClass(String domainName, DateTime date) {
-    return getDomainFeeClass(domainName, date);
+  public Optional<String> getFeeClass(String domainName, DateTime dateTime) {
+    return getDomainFeeClass(domainName, dateTime);
   }
 
+  /** Returns the domain create cost with allocation-token-related discounts applied. */
   private Money getDomainCreateCostWithDiscount(
-      String domainName, DateTime date, int years, Optional<AllocationToken> allocationToken)
+      DomainPrices domainPrices, int years, Optional<AllocationToken> allocationToken)
       throws EppException {
-    DomainPrices domainPrices = PricingEngineProxy.getPricesForDomainName(domainName, date);
     if (allocationToken.isPresent()
         && allocationToken.get().getDiscountFraction() != 0.0
         && domainPrices.isPremium()) {

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -211,8 +211,7 @@ public final class DomainRenewFlow implements TransactionalFlow {
             BeforeResponseParameters.newBuilder()
                 .setDomain(newDomain)
                 .setResData(DomainRenewData.create(targetId, newExpirationTime))
-                .setResponseExtensions(
-                    createResponseExtensions(feesAndCredits.getTotalCost(), feeRenew))
+                .setResponseExtensions(createResponseExtensions(feesAndCredits, feeRenew))
                 .build());
     return responseBuilder
         .setResData(responseData.resData())
@@ -270,14 +269,19 @@ public final class DomainRenewFlow implements TransactionalFlow {
   }
 
   private ImmutableList<FeeTransformResponseExtension> createResponseExtensions(
-      Money renewCost, Optional<FeeRenewCommandExtension> feeRenew) {
+      FeesAndCredits feesAndCredits, Optional<FeeRenewCommandExtension> feeRenew) {
     return feeRenew.isPresent()
         ? ImmutableList.of(
             feeRenew
                 .get()
                 .createResponseBuilder()
-                .setCurrency(renewCost.getCurrencyUnit())
-                .setFees(ImmutableList.of(Fee.create(renewCost.getAmount(), FeeType.RENEW)))
+                .setCurrency(feesAndCredits.getCurrency())
+                .setFees(
+                    ImmutableList.of(
+                        Fee.create(
+                            feesAndCredits.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RENEW))))
                 .build())
         : ImmutableList.of();
   }

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -143,24 +143,30 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     verifyRestoreAllowed(command, existingDomain, feeUpdate, feesAndCredits, now);
     HistoryEntry historyEntry = buildHistoryEntry(existingDomain, now);
     ImmutableSet.Builder<ImmutableObject> entitiesToSave = new ImmutableSet.Builder<>();
-    entitiesToSave.addAll(
-        createRestoreAndRenewBillingEvents(
-            historyEntry, feesAndCredits.getRestoreCost(), feesAndCredits.getRenewCost(), now));
-    // We don't preserve the original expiration time of the domain when we restore, since doing so
-    // would require us to know if they received a grace period refund when they deleted the domain,
-    // and to charge them for that again. Instead, we just say that all restores get a fresh year of
-    // registration and bill them for that accordingly.
-    DateTime newExpirationTime = now.plusYears(1);
-    BillingEvent.Recurring autorenewEvent = newAutorenewBillingEvent(existingDomain)
-        .setEventTime(newExpirationTime)
-        .setRecurrenceEndTime(END_OF_TIME)
-        .setParent(historyEntry)
-        .build();
-    PollMessage.Autorenew autorenewPollMessage = newAutorenewPollMessage(existingDomain)
-        .setEventTime(newExpirationTime)
-        .setAutorenewEndTime(END_OF_TIME)
-        .setParent(historyEntry)
-        .build();
+
+    // Restore the expiration time on the deleted domain, except if that's already passed, then add
+    // a year and bill for it immediately, with no grace period.
+    DateTime newExpirationTime = existingDomain.getRegistrationExpirationTime();
+    if (newExpirationTime.isBefore(now)) {
+      entitiesToSave.add(createRenewBillingEvent(historyEntry, feesAndCredits.getRenewCost(), now));
+      newExpirationTime = newExpirationTime.plusYears(1);
+    }
+    // Always bill for the restore itself.
+    entitiesToSave.add(
+        createRestoreBillingEvent(historyEntry, feesAndCredits.getRestoreCost(), now));
+
+    BillingEvent.Recurring autorenewEvent =
+        newAutorenewBillingEvent(existingDomain)
+            .setEventTime(newExpirationTime)
+            .setRecurrenceEndTime(END_OF_TIME)
+            .setParent(historyEntry)
+            .build();
+    PollMessage.Autorenew autorenewPollMessage =
+        newAutorenewPollMessage(existingDomain)
+            .setEventTime(newExpirationTime)
+            .setAutorenewEndTime(END_OF_TIME)
+            .setParent(historyEntry)
+            .build();
     DomainBase newDomain =
         performRestore(
             existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId);
@@ -170,9 +176,7 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     ofy().delete().key(existingDomain.getDeletePollMessage());
     dnsQueue.addDomainRefreshTask(existingDomain.getFullyQualifiedDomainName());
     return responseBuilder
-        .setExtensions(
-            createResponseExtensions(
-                feesAndCredits.getRestoreCost(), feesAndCredits.getRenewCost(), feeUpdate))
+        .setExtensions(createResponseExtensions(feesAndCredits, feeUpdate))
         .build();
   }
 
@@ -212,18 +216,6 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
     validateFeeChallenge(targetId, now, feeUpdate, feesAndCredits);
   }
 
-  private ImmutableSet<BillingEvent.OneTime> createRestoreAndRenewBillingEvents(
-      HistoryEntry historyEntry, Money restoreCost, Money renewCost, DateTime now) {
-    // Bill for the restore.
-    BillingEvent.OneTime restoreEvent = createRestoreBillingEvent(historyEntry, restoreCost, now);
-    // Create a new autorenew billing event and poll message starting at the new expiration time.
-    // Also bill for the 1 year cost of a domain renew. This is to avoid registrants being able to
-    // game the system for premium names by renewing, deleting, and then restoring to get a free
-    // year. Note that this billing event has no grace period; it is effective immediately.
-    BillingEvent.OneTime renewEvent = createRenewBillingEvent(historyEntry, renewCost, now);
-    return ImmutableSet.of(restoreEvent, renewEvent);
-  }
-
   private static DomainBase performRestore(
       DomainBase existingDomain,
       DateTime newExpirationTime,
@@ -271,17 +263,23 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
   }
 
   private static ImmutableList<FeeTransformResponseExtension> createResponseExtensions(
-      Money restoreCost, Money renewCost, Optional<FeeUpdateCommandExtension> feeUpdate) {
+      FeesAndCredits feesAndCredits, Optional<FeeUpdateCommandExtension> feeUpdate) {
     return feeUpdate.isPresent()
         ? ImmutableList.of(
             feeUpdate
                 .get()
                 .createResponseBuilder()
-                .setCurrency(restoreCost.getCurrencyUnit())
+                .setCurrency(feesAndCredits.getCurrency())
                 .setFees(
                     ImmutableList.of(
-                        Fee.create(restoreCost.getAmount(), FeeType.RESTORE),
-                        Fee.create(renewCost.getAmount(), FeeType.RENEW)))
+                        Fee.create(
+                            feesAndCredits.getRestoreCost().getAmount(),
+                            FeeType.RESTORE,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RESTORE)),
+                        Fee.create(
+                            feesAndCredits.getRenewCost().getAmount(),
+                            FeeType.RENEW,
+                            feesAndCredits.hasPremiumFeesOfType(FeeType.RENEW))))
                 .build())
         : ImmutableList.of();
   }

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -214,7 +214,7 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
     ofy().save().entities(entitiesToSave.build());
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(
             targetId, newDomain.getTransferData(), newDomain.getRegistrationExpirationTime()))

core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java

@@ -110,7 +110,7 @@ public final class DomainTransferCancelFlow implements TransactionalFlow {
     updateAutorenewRecurrenceEndTime(existingDomain, END_OF_TIME);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newDomain.getTransferData(), null))
         .build();

core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java

@@ -112,7 +112,7 @@ public final class DomainTransferRejectFlow implements TransactionalFlow {
     updateAutorenewRecurrenceEndTime(existingDomain, END_OF_TIME);
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
-    ofy().delete().keys(existingDomain.getTransferData().getServerApproveEntities());
+    tm().delete(existingDomain.getTransferData().getServerApproveEntities());
     return responseBuilder
         .setResData(createTransferResponse(targetId, newDomain.getTransferData(), null))
         .build();

core/src/main/java/google/registry/flows/domain/DomainTransferUtils.java

@@ -20,7 +20,6 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
@@ -37,6 +36,7 @@ import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import org.joda.money.Money;
@@ -52,37 +52,34 @@ public final class DomainTransferUtils {
       TransferData.Builder transferDataBuilder,
       ImmutableSet<TransferServerApproveEntity> serverApproveEntities,
       Period transferPeriod) {
-    ImmutableSet.Builder<Key<? extends TransferServerApproveEntity>> serverApproveEntityKeys =
+    ImmutableSet.Builder<VKey<? extends TransferServerApproveEntity>> serverApproveEntityKeys =
         new ImmutableSet.Builder<>();
     for (TransferServerApproveEntity entity : serverApproveEntities) {
-      serverApproveEntityKeys.add(Key.create(entity));
+      serverApproveEntityKeys.add(entity.createVKey());
     }
     if (transferPeriod.getValue() != 0) {
       // Unless superuser sets period to 0, add a transfer billing event.
       transferDataBuilder.setServerApproveBillingEvent(
-          Key.create(
-              serverApproveEntities
-                  .stream()
-                  .filter(BillingEvent.OneTime.class::isInstance)
-                  .map(BillingEvent.OneTime.class::cast)
-                  .collect(onlyElement())));
+          serverApproveEntities.stream()
+              .filter(BillingEvent.OneTime.class::isInstance)
+              .map(BillingEvent.OneTime.class::cast)
+              .collect(onlyElement())
+              .createVKey());
     }
     return transferDataBuilder
         .setTransferStatus(TransferStatus.PENDING)
         .setServerApproveAutorenewEvent(
-            Key.create(
-                serverApproveEntities
-                    .stream()
-                    .filter(BillingEvent.Recurring.class::isInstance)
-                    .map(BillingEvent.Recurring.class::cast)
-                    .collect(onlyElement())))
+            serverApproveEntities.stream()
+                .filter(BillingEvent.Recurring.class::isInstance)
+                .map(BillingEvent.Recurring.class::cast)
+                .collect(onlyElement())
+                .createVKey())
         .setServerApproveAutorenewPollMessage(
-            Key.create(
-                serverApproveEntities
-                    .stream()
-                    .filter(PollMessage.Autorenew.class::isInstance)
-                    .map(PollMessage.Autorenew.class::cast)
-                    .collect(onlyElement())))
+            serverApproveEntities.stream()
+                .filter(PollMessage.Autorenew.class::isInstance)
+                .map(PollMessage.Autorenew.class::cast)
+                .collect(onlyElement())
+                .createVKey())
         .setServerApproveEntities(serverApproveEntityKeys.build())
         .setTransferPeriod(transferPeriod)
         .build();

core/src/main/java/google/registry/flows/domain/FeesAndCredits.java

@@ -15,6 +15,7 @@
 package google.registry.flows.domain;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.flows.domain.DomainFlowUtils.zeroInCurrency;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
@@ -27,6 +28,7 @@ import google.registry.model.domain.fee.BaseFee;
 import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Credit;
 import google.registry.model.domain.fee.Fee;
+import java.math.BigDecimal;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 
@@ -39,26 +41,30 @@ public class FeesAndCredits extends ImmutableObject implements Buildable {
   private ImmutableList<Credit> credits;
 
   private Money getTotalCostForType(FeeType type) {
-    Money result = Money.zero(currency);
     checkArgumentNotNull(type);
-    for (Fee fee : fees) {
-      if (fee.getType() == type) {
-        result = result.plus(fee.getCost());
-      }
-    }
-    return result;
+    return Money.of(
+        currency,
+        fees.stream()
+            .filter(f -> f.getType() == type)
+            .map(BaseFee::getCost)
+            .reduce(zeroInCurrency(currency), BigDecimal::add));
+  }
+
+  public boolean hasPremiumFeesOfType(FeeType type) {
+    return fees.stream().filter(f -> f.getType() == type).anyMatch(BaseFee::isPremium);
   }
 
   /** Returns the total cost of all fees and credits for the event. */
   public Money getTotalCost() {
-    Money result = Money.zero(currency);
-    for (Fee fee : fees) {
-      result = result.plus(fee.getCost());
-    }
-    for (Credit credit : credits) {
-      result = result.plus(credit.getCost());
-    }
-    return result;
+    return Money.of(
+        currency,
+        Streams.concat(fees.stream(), credits.stream())
+            .map(BaseFee::getCost)
+            .reduce(zeroInCurrency(currency), BigDecimal::add));
+  }
+
+  public boolean hasAnyPremiumFees() {
+    return fees.stream().anyMatch(BaseFee::isPremium);
   }
 
   /** Returns the create cost for the event. */

core/src/main/java/google/registry/flows/host/HostCreateFlow.java

@@ -128,7 +128,7 @@ public final class HostCreateFlow implements TransactionalFlow {
             .setFullyQualifiedHostName(targetId)
             .setInetAddresses(command.getInetAddresses())
             .setRepoId(createRepoId(ObjectifyService.allocateId(), roidSuffix))
-            .setSuperordinateDomain(superordinateDomain.map(Key::create).orElse(null))
+            .setSuperordinateDomain(superordinateDomain.map(DomainBase::createVKey).orElse(null))
             .build();
     historyBuilder
         .setType(HistoryEntry.Type.HOST_CREATE)

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -96,8 +96,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
       // the client id, needs to be read off of it.
       EppResource owningResource =
           existingHost.isSubordinate()
-              ? ofy().load().key(existingHost.getSuperordinateDomain()).now()
-                  .cloneProjectedAtTime(now)
+              ? tm().load(existingHost.getSuperordinateDomain()).cloneProjectedAtTime(now)
               : existingHost;
       verifyResourceOwnership(clientId, owningResource);
     }

core/src/main/java/google/registry/flows/host/HostInfoFlow.java

@@ -18,7 +18,7 @@ import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.ResourceFlowUtils.loadAndVerifyExistence;
 import static google.registry.flows.host.HostFlowUtils.validateHostName;
 import static google.registry.model.EppResourceUtils.isLinked;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
@@ -77,7 +77,7 @@ public final class HostInfoFlow implements Flow {
     // there is no superordinate domain, the host's own values for these fields will be correct.
     if (host.isSubordinate()) {
       DomainBase superordinateDomain =
-          ofy().load().key(host.getSuperordinateDomain()).now().cloneProjectedAtTime(now);
+          tm().load(host.getSuperordinateDomain()).cloneProjectedAtTime(now);
       hostInfoDataBuilder
           .setCurrentSponsorClientId(superordinateDomain.getCurrentSponsorClientId())
           .setLastTransferTime(host.computeLastTransferTime(superordinateDomain));

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -60,6 +60,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
+import google.registry.persistence.VKey;
 import java.util.Objects;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -136,9 +137,10 @@ public final class HostUpdateFlow implements TransactionalFlow {
     boolean isHostRename = suppliedNewHostName != null;
     String oldHostName = targetId;
     String newHostName = firstNonNull(suppliedNewHostName, oldHostName);
-    DomainBase oldSuperordinateDomain = existingHost.isSubordinate()
-        ? ofy().load().key(existingHost.getSuperordinateDomain()).now().cloneProjectedAtTime(now)
-        : null;
+    DomainBase oldSuperordinateDomain =
+        existingHost.isSubordinate()
+            ? tm().load(existingHost.getSuperordinateDomain()).cloneProjectedAtTime(now)
+            : null;
     // Note that lookupSuperordinateDomain calls cloneProjectedAtTime on the domain for us.
     Optional<DomainBase> newSuperordinateDomain =
         lookupSuperordinateDomain(validateHostName(newHostName), now);
@@ -153,8 +155,8 @@ public final class HostUpdateFlow implements TransactionalFlow {
     AddRemove remove = command.getInnerRemove();
     checkSameValuesNotAddedAndRemoved(add.getStatusValues(), remove.getStatusValues());
     checkSameValuesNotAddedAndRemoved(add.getInetAddresses(), remove.getInetAddresses());
-    Key<DomainBase> newSuperordinateDomainKey =
-        newSuperordinateDomain.map(Key::create).orElse(null);
+    VKey<DomainBase> newSuperordinateDomainKey =
+        newSuperordinateDomain.map(DomainBase::createVKey).orElse(null);
     // If the superordinateDomain field is changing, set the lastSuperordinateChange to now.
     DateTime lastSuperordinateChange =
         Objects.equals(newSuperordinateDomainKey, existingHost.getSuperordinateDomain())
@@ -280,28 +282,28 @@ public final class HostUpdateFlow implements TransactionalFlow {
     if (existingHost.isSubordinate()
         && newHost.isSubordinate()
         && Objects.equals(
-            existingHost.getSuperordinateDomain(),
-            newHost.getSuperordinateDomain())) {
-      ofy().save().entity(
-          ofy().load().key(existingHost.getSuperordinateDomain()).now().asBuilder()
-              .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
-              .addSubordinateHost(newHost.getFullyQualifiedHostName())
-              .build());
+            existingHost.getSuperordinateDomain(), newHost.getSuperordinateDomain())) {
+      tm().saveNewOrUpdate(
+              tm().load(existingHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
+                  .addSubordinateHost(newHost.getFullyQualifiedHostName())
+                  .build());
       return;
     }
     if (existingHost.isSubordinate()) {
-      ofy().save().entity(
-          ofy().load().key(existingHost.getSuperordinateDomain()).now()
-              .asBuilder()
-              .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
-              .build());
+      tm().saveNewOrUpdate(
+              tm().load(existingHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .removeSubordinateHost(existingHost.getFullyQualifiedHostName())
+                  .build());
     }
     if (newHost.isSubordinate()) {
-      ofy().save().entity(
-          ofy().load().key(newHost.getSuperordinateDomain()).now()
-              .asBuilder()
-              .addSubordinateHost(newHost.getFullyQualifiedHostName())
-              .build());
+      tm().saveNewOrUpdate(
+              tm().load(newHost.getSuperordinateDomain())
+                  .asBuilder()
+                  .addSubordinateHost(newHost.getFullyQualifiedHostName())
+                  .build());
     }
   }
 

core/src/main/java/google/registry/model/EppResource.java

@@ -68,11 +68,11 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /** The ID of the registrar that is currently sponsoring this resource. */
   @Index
-  @Column(nullable = false)
+  @Column(name = "currentSponsorRegistrarId", nullable = false)
   String currentSponsorClientId;
 
   /** The ID of the registrar that created this resource. */
-  @Column(nullable = false)
+  @Column(name = "creationRegistrarId", nullable = false)
   String creationClientId;
 
   /**
@@ -82,6 +82,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * edits; it only includes EPP-visible modifications such as {@literal <update>}. Can be null if
    * the resource has never been modified.
    */
+  @Column(name = "lastEppUpdateRegistrarId")
   String lastEppUpdateClientId;
 
   /** The time when this resource was created. */

core/src/main/java/google/registry/model/ResourceTransferUtils.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -114,7 +115,7 @@ public final class ResourceTransferUtils {
             R resource, R newResource, DateTime now, HistoryEntry historyEntry) {
     if (resource.getStatusValues().contains(StatusValue.PENDING_TRANSFER)) {
       TransferData oldTransferData = resource.getTransferData();
-      ofy().delete().keys(oldTransferData.getServerApproveEntities());
+      tm().delete(oldTransferData.getServerApproveEntities());
       ofy()
           .save()
           .entity(

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -116,7 +116,7 @@ public abstract class BillingEvent extends ImmutableObject
 
   /** The registrar to bill. */
   @Index
-  @Column(nullable = false)
+  @Column(name = "registrarId", nullable = false)
   String clientId;
 
   /** Revision id of the entry in DomainHistory table that ths bill belongs to. */
@@ -267,7 +267,7 @@ public abstract class BillingEvent extends ImmutableObject
   @javax.persistence.Entity(name = "BillingEvent")
   @javax.persistence.Table(
       indexes = {
-        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "registrarId"),
         @javax.persistence.Index(columnList = "eventTime"),
         @javax.persistence.Index(columnList = "billingTime"),
         @javax.persistence.Index(columnList = "syntheticCreationTime"),
@@ -345,6 +345,7 @@ public abstract class BillingEvent extends ImmutableObject
       return Optional.ofNullable(allocationToken);
     }
 
+    @Override
     public VKey<OneTime> createVKey() {
       return VKey.createOfy(getClass(), Key.create(this));
     }
@@ -439,7 +440,7 @@ public abstract class BillingEvent extends ImmutableObject
   @javax.persistence.Entity(name = "BillingRecurrence")
   @javax.persistence.Table(
       indexes = {
-        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "registrarId"),
         @javax.persistence.Index(columnList = "eventTime"),
         @javax.persistence.Index(columnList = "recurrenceEndTime"),
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
@@ -482,6 +483,7 @@ public abstract class BillingEvent extends ImmutableObject
       return recurrenceTimeOfYear;
     }
 
+    @Override
     public VKey<Recurring> createVKey() {
       return VKey.createOfy(getClass(), Key.create(this));
     }
@@ -529,7 +531,7 @@ public abstract class BillingEvent extends ImmutableObject
   @javax.persistence.Entity(name = "BillingCancellation")
   @javax.persistence.Table(
       indexes = {
-        @javax.persistence.Index(columnList = "clientId"),
+        @javax.persistence.Index(columnList = "registrarId"),
         @javax.persistence.Index(columnList = "eventTime"),
         @javax.persistence.Index(columnList = "billingTime")
       })
@@ -603,6 +605,7 @@ public abstract class BillingEvent extends ImmutableObject
       return builder.build();
     }
 
+    @Override
     public VKey<Cancellation> createVKey() {
       return VKey.createOfy(getClass(), Key.create(this));
     }
@@ -682,6 +685,11 @@ public abstract class BillingEvent extends ImmutableObject
       return new Builder(clone(this));
     }
 
+    @Override
+    public VKey<Modification> createVKey() {
+      return VKey.createOfy(this.getClass(), Key.create(this));
+    }
+
     /**
      * Create a new Modification billing event which is a refund of the given OneTime billing event
      * and that is parented off the given HistoryEntry.

core/src/main/java/google/registry/model/common/Cursor.java

@@ -35,9 +35,9 @@ import java.util.List;
 import org.joda.time.DateTime;
 
 /**
- * Shared entity for date cursors. This type supports both "scoped" cursors (i.e. per resource
- * of a given type, such as a TLD) and global (i.e. one per environment) cursors, defined internally
- * as scoped on {@link EntityGroupRoot}.
+ * Shared entity for date cursors. This type supports both "scoped" cursors (i.e. per resource of a
+ * given type, such as a TLD) and global (i.e. one per environment) cursors, defined internally as
+ * scoped on {@link EntityGroupRoot}.
  */
 @Entity
 public class Cursor extends ImmutableObject implements DatastoreEntity {

core/src/main/java/google/registry/model/common/EntityGroupRoot.java

@@ -27,14 +27,14 @@ import google.registry.schema.replay.SqlEntity;
  * reasons.
  *
  * <p>This exists as a storage place for common configuration options and global settings that
- * aren't updated too frequently.  Entities in this entity group are usually cached upon load.  The
+ * aren't updated too frequently. Entities in this entity group are usually cached upon load. The
  * reason this common entity group exists is because it enables strongly consistent queries and
- * updates across this seldomly updated data.  This shared entity group also helps cut down on
- * a potential ballooning in the number of entity groups enlisted in transactions.
+ * updates across this seldomly updated data. This shared entity group also helps cut down on a
+ * potential ballooning in the number of entity groups enlisted in transactions.
  *
- * <p>Historically, each TLD used to have a separate namespace, and all entities for a TLD were in
- * a single EntityGroupRoot for that TLD.  Hence why there was a "cross-tld" entity group -- it was
- * the entity group for the single namespace where global data applicable for all TLDs lived.
+ * <p>Historically, each TLD used to have a separate namespace, and all entities for a TLD were in a
+ * single EntityGroupRoot for that TLD. Hence why there was a "cross-tld" entity group -- it was the
+ * entity group for the single namespace where global data applicable for all TLDs lived.
  */
 @Entity
 public class EntityGroupRoot extends BackupGroupRoot implements DatastoreEntity {

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -41,7 +41,6 @@ import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Embedded;
-import javax.persistence.Transient;
 import javax.xml.bind.annotation.XmlElement;
 import org.joda.time.DateTime;
 
@@ -57,7 +56,7 @@ import org.joda.time.DateTime;
     name = "Contact",
     indexes = {
       @javax.persistence.Index(columnList = "creationTime"),
-      @javax.persistence.Index(columnList = "currentSponsorClientId"),
+      @javax.persistence.Index(columnList = "currentSponsorRegistrarId"),
       @javax.persistence.Index(columnList = "deletionTime"),
       @javax.persistence.Index(columnList = "contactId", unique = true),
       @javax.persistence.Index(columnList = "searchName")
@@ -171,8 +170,7 @@ public class ContactResource extends EppResource
   ContactAuthInfo authInfo;
 
   /** Data about any pending or past transfers on this contact. */
-  // TODO(b/153363295): Figure out how to persist transfer data
-  @Transient TransferData transferData;
+  TransferData transferData;
 
   /**
    * The time that this resource was last transferred.

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -66,6 +66,7 @@ import google.registry.model.registry.Registry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.util.CollectionUtils;
 import java.util.HashSet;
@@ -101,11 +102,12 @@ import org.joda.time.Interval;
     name = "Domain",
     indexes = {
       @javax.persistence.Index(columnList = "creationTime"),
-      @javax.persistence.Index(columnList = "currentSponsorClientId"),
+      @javax.persistence.Index(columnList = "currentSponsorRegistrarId"),
       @javax.persistence.Index(columnList = "deletionTime"),
       @javax.persistence.Index(columnList = "fullyQualifiedDomainName"),
       @javax.persistence.Index(columnList = "tld")
     })
+@WithStringVKey
 @ExternalMessagingName("domain")
 public class DomainBase extends EppResource
     implements DatastoreAndSqlEntity, ForeignKeyedEppResource, ResourceWithTransferData {
@@ -251,7 +253,7 @@ public class DomainBase extends EppResource
   String smdId;
 
   /** Data about any pending or past transfers on this domain. */
-  @Transient TransferData transferData;
+  TransferData transferData;
 
   /**
    * The time that this resource was last transferred.
@@ -436,8 +438,14 @@ public class DomainBase extends EppResource
               .setRegistrationExpirationTime(expirationDate)
               // Set the speculatively-written new autorenew events as the domain's autorenew
               // events.
-              .setAutorenewBillingEvent(transferData.getServerApproveAutorenewEvent())
-              .setAutorenewPollMessage(transferData.getServerApproveAutorenewPollMessage());
+              .setAutorenewBillingEvent(
+                  transferData.getServerApproveAutorenewEvent() == null
+                      ? null
+                      : transferData.getServerApproveAutorenewEvent().getOfyKey())
+              .setAutorenewPollMessage(
+                  transferData.getServerApproveAutorenewPollMessage() == null
+                      ? null
+                      : transferData.getServerApproveAutorenewPollMessage().getOfyKey());
       if (transferData.getTransferPeriod().getValue() == 1) {
         // Set the grace period using a key to the prescheduled transfer billing event.  Not using
         // GracePeriod.forBillingEvent() here in order to avoid the actual Datastore fetch.
@@ -448,7 +456,9 @@ public class DomainBase extends EppResource
                     transferExpirationTime.plus(
                         Registry.get(getTld()).getTransferGracePeriodLength()),
                     transferData.getGainingClientId(),
-                    transferData.getServerApproveBillingEvent())));
+                    transferData.getServerApproveBillingEvent() == null
+                        ? null
+                        : transferData.getServerApproveBillingEvent().getOfyKey())));
       } else {
         // There won't be a billing event, so we don't need a grace period
         builder.setGracePeriods(ImmutableSet.of());
@@ -615,7 +625,8 @@ public class DomainBase extends EppResource
 
   @Override
   public VKey<DomainBase> createVKey() {
-    return VKey.create(DomainBase.class, getRepoId(), Key.create(this));
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.createOfy(DomainBase.class, Key.create(this));
   }
 
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -24,6 +24,7 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import org.joda.time.DateTime;
@@ -51,6 +52,7 @@ public class GracePeriod extends ImmutableObject {
   DateTime expirationTime;
 
   /** The registrar to bill. */
+  @Column(name = "registrarId")
   String clientId;
 
   /**

core/src/main/java/google/registry/model/domain/Period.java

@@ -16,6 +16,8 @@ package google.registry.model.domain;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlEnumValue;
 import javax.xml.bind.annotation.XmlValue;
@@ -25,6 +27,7 @@ import javax.xml.bind.annotation.XmlValue;
 @javax.persistence.Embeddable
 public class Period extends ImmutableObject {
 
+  @Enumerated(EnumType.STRING)
   @XmlAttribute
   Unit unit;
 

core/src/main/java/google/registry/model/domain/fee/BaseFee.java

@@ -104,6 +104,8 @@ public abstract class BaseFee extends ImmutableObject {
 
   @XmlTransient Range<DateTime> validDateRange;
 
+  @XmlTransient boolean isPremium;
+
   public String getDescription() {
     return description;
   }
@@ -120,6 +122,11 @@ public abstract class BaseFee extends ImmutableObject {
     return firstNonNull(refundable, true);
   }
 
+  /** Returns whether the fee in question is a premium price. */
+  public boolean isPremium() {
+    return isPremium;
+  }
+
   /**
    * According to the fee extension specification, a fee must always be non-negative, while a credit
    * must always be negative. Essentially, they are the same thing, just with different sign.

core/src/main/java/google/registry/model/domain/fee/Fee.java

@@ -31,25 +31,33 @@ import org.joda.time.DateTime;
 public class Fee extends BaseFee {
 
   /** Creates a Fee for the given cost and type with the default description. */
-  public static Fee create(BigDecimal cost, FeeType type, Object... descriptionArgs) {
+  public static Fee create(
+      BigDecimal cost, FeeType type, boolean isPremium, Object... descriptionArgs) {
     checkArgumentNotNull(type, "Must specify the type of the fee");
-    return createWithCustomDescription(cost, type, type.renderDescription(descriptionArgs));
+    return createWithCustomDescription(
+        cost, type, isPremium, type.renderDescription(descriptionArgs));
   }
 
   /** Creates a Fee for the given cost, type, and valid date range with the default description. */
   public static Fee create(
-      BigDecimal cost, FeeType type, Range<DateTime> validDateRange, Object... descriptionArgs) {
-    Fee instance = create(cost, type, descriptionArgs);
+      BigDecimal cost,
+      FeeType type,
+      boolean isPremium,
+      Range<DateTime> validDateRange,
+      Object... descriptionArgs) {
+    Fee instance = create(cost, type, isPremium, descriptionArgs);
     instance.validDateRange = validDateRange;
     return instance;
   }
 
   /** Creates a Fee for the given cost and type with a custom description. */
-  public static Fee createWithCustomDescription(BigDecimal cost, FeeType type, String description) {
+  private static Fee createWithCustomDescription(
+      BigDecimal cost, FeeType type, boolean isPremium, String description) {
     Fee instance = new Fee();
     instance.cost = checkNotNull(cost);
-    checkArgument(instance.cost.signum() >= 0);
+    checkArgument(instance.cost.signum() >= 0, "Cost must be a positive number");
     instance.type = checkNotNull(type);
+    instance.isPremium = isPremium;
     instance.description = description;
     return instance;
   }

core/src/main/java/google/registry/model/domain/fee11/FeeCheckCommandExtensionV11.java

@@ -56,7 +56,7 @@ public class FeeCheckCommandExtensionV11 extends ImmutableObject
   /** The period to check. */
   Period period;
 
-  /** The class to check. */
+  /** The fee class to check. */
   @XmlElement(name = "class")
   String feeClass;
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -40,7 +40,6 @@ import java.net.InetAddress;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
-import javax.persistence.ElementCollection;
 import org.joda.time.DateTime;
 
 /**
@@ -70,13 +69,13 @@ public class HostResource extends EppResource
   String fullyQualifiedHostName;
 
   /** IP Addresses for this host. Can be null if this is an external host. */
-  @Index @ElementCollection Set<InetAddress> inetAddresses;
+  @Index Set<InetAddress> inetAddresses;
 
   /** The superordinate domain of this host, or null if this is an external host. */
   @Index
   @IgnoreSave(IfNull.class)
   @DoNotHydrate
-  Key<DomainBase> superordinateDomain;
+  VKey<DomainBase> superordinateDomain;
 
   /**
    * The time that this resource was last transferred.
@@ -98,7 +97,7 @@ public class HostResource extends EppResource
     return fullyQualifiedHostName;
   }
 
-  public Key<DomainBase> getSuperordinateDomain() {
+  public VKey<DomainBase> getSuperordinateDomain() {
     return superordinateDomain;
   }
 
@@ -145,18 +144,18 @@ public class HostResource extends EppResource
    *
    * <p>If the host is not subordinate the domain can be null and we just return last transfer time.
    *
-   * @param superordinateDomain the loaded superordinate domain, which must match the key in
-   *     the {@link #superordinateDomain} field. Passing it as a parameter allows the caller to
-   *     control the degree of consistency used to load it.
+   * @param superordinateDomain the loaded superordinate domain, which must match the key in the
+   *     {@link #superordinateDomain} field. Passing it as a parameter allows the caller to control
+   *     the degree of consistency used to load it.
    */
-   public DateTime computeLastTransferTime(@Nullable DomainBase superordinateDomain) {
+  public DateTime computeLastTransferTime(@Nullable DomainBase superordinateDomain) {
     if (!isSubordinate()) {
       checkArgument(superordinateDomain == null);
       return getLastTransferTime();
     }
     checkArgument(
         superordinateDomain != null
-            && Key.create(superordinateDomain).equals(getSuperordinateDomain()));
+            && superordinateDomain.createVKey().equals(getSuperordinateDomain()));
     DateTime lastSuperordinateChange =
         Optional.ofNullable(getLastSuperordinateChange()).orElse(getCreationTime());
     DateTime lastTransferOfCurrentSuperordinate =
@@ -207,7 +206,7 @@ public class HostResource extends EppResource
           difference(getInstance().getInetAddresses(), inetAddresses)));
     }
 
-    public Builder setSuperordinateDomain(Key<DomainBase> superordinateDomain) {
+    public Builder setSuperordinateDomain(VKey<DomainBase> superordinateDomain) {
       getInstance().superordinateDomain = superordinateDomain;
       return this;
     }

core/src/main/java/google/registry/model/ofy/CommitLogBucket.java

@@ -44,10 +44,9 @@ import org.joda.time.DateTime;
 /**
  * Root for a random commit log bucket.
  *
- * <p>This is used to shard {@link CommitLogManifest} objects into
- * {@link RegistryConfig#getCommitLogBucketCount() N} entity groups. This increases
- * transaction throughput, while maintaining the ability to perform strongly-consistent ancestor
- * queries.
+ * <p>This is used to shard {@link CommitLogManifest} objects into {@link
+ * RegistryConfig#getCommitLogBucketCount() N} entity groups. This increases transaction throughput,
+ * while maintaining the ability to perform strongly-consistent ancestor queries.
  *
  * @see <a href="https://cloud.google.com/appengine/articles/scaling/contention">Avoiding Datastore
  *     contention</a>

core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java

@@ -38,11 +38,11 @@ import org.joda.time.DateTime;
  * Entity representing a point-in-time consistent view of Datastore, based on commit logs.
  *
  * <p>Conceptually, this entity consists of two pieces of information: the checkpoint "wall" time
- * and a set of bucket checkpoint times.  The former is the ID for this checkpoint (constrained
- * to be unique upon checkpoint creation) and also represents the approximate wall time of the
- * consistent Datastore view this checkpoint represents.  The latter is really a mapping from
- * bucket ID to timestamp, where the timestamp dictates the upper bound (inclusive) on commit logs
- * from that bucket to include when restoring Datastore to this checkpoint.
+ * and a set of bucket checkpoint times. The former is the ID for this checkpoint (constrained to be
+ * unique upon checkpoint creation) and also represents the approximate wall time of the consistent
+ * Datastore view this checkpoint represents. The latter is really a mapping from bucket ID to
+ * timestamp, where the timestamp dictates the upper bound (inclusive) on commit logs from that
+ * bucket to include when restoring Datastore to this checkpoint.
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)

core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java

@@ -28,9 +28,7 @@ import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import org.joda.time.DateTime;
 
-/**
- * Singleton parent entity for all commit log checkpoints.
- */
+/** Singleton parent entity for all commit log checkpoints. */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
 public class CommitLogCheckpointRoot extends ImmutableObject implements DatastoreEntity {

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -43,6 +43,7 @@ import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferResponse.ContactTransferResponse;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
+import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
 import java.util.List;
 import java.util.Optional;
@@ -150,6 +151,9 @@ public abstract class PollMessage extends ImmutableObject
 
   public abstract ImmutableList<ResponseData> getResponseData();
 
+  @Override
+  public abstract VKey<? extends PollMessage> createVKey();
+
   /** Override Buildable.asBuilder() to give this method stronger typing. */
   @Override
   public abstract Builder<?, ?> asBuilder();
@@ -291,6 +295,11 @@ public abstract class PollMessage extends ImmutableObject
     @Column(name = "transfer_response_contact_id")
     String contactId;
 
+    @Override
+    public VKey<OneTime> createVKey() {
+      return VKey.createOfy(this.getClass(), Key.create(this));
+    }
+
     @Override
     public Builder asBuilder() {
       return new Builder(clone(this));
@@ -387,6 +396,11 @@ public abstract class PollMessage extends ImmutableObject
       return autorenewEndTime;
     }
 
+    @Override
+    public VKey<Autorenew> createVKey() {
+      return VKey.createOfy(this.getClass(), Key.create(this));
+    }
+
     @Override
     public ImmutableList<ResponseData> getResponseData() {
       // Note that the event time is when the auto-renew occured, so the expiration time in the

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -236,7 +236,7 @@ public class Registrar extends ImmutableObject
    */
   @Id
   @javax.persistence.Id
-  @Column(name = "client_id", nullable = false)
+  @Column(name = "registrarId", nullable = false)
   String clientIdentifier;
 
   /**

core/src/main/java/google/registry/model/registry/Registry.java

@@ -579,6 +579,8 @@ public class Registry extends ImmutableObject implements Buildable {
     return Fee.create(
         eapFeeSchedule.getValueAtTime(now).getAmount(),
         FeeType.EAP,
+        // An EAP fee counts as premium so the domain's overall Fee doesn't show as standard-priced.
+        true,
         validPeriod,
         validPeriod.upperEndpoint());
   }

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -248,7 +248,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   }
 
   /**
-   * A premium list entry entity, persisted to Datastore.  Each instance represents the price of a
+   * A premium list entry entity, persisted to Datastore. Each instance represents the price of a
    * single label on a given TLD.
    */
   @ReportedOn

core/src/main/java/google/registry/model/registry/label/ReservedList.java

@@ -62,8 +62,8 @@ import org.joda.time.DateTime;
  */
 @Entity
 public final class ReservedList
-    extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry> implements
-    DatastoreEntity {
+    extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry>
+    implements DatastoreEntity {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -327,8 +327,8 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
   }
 
   /**
-   * Serves as the coordinating claims list singleton linking to the {@link ClaimsListRevision}
-   * that is live.
+   * Serves as the coordinating claims list singleton linking to the {@link ClaimsListRevision} that
+   * is live.
    */
   @Entity
   @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)

core/src/main/java/google/registry/model/transfer/BaseTransferObject.java

@@ -16,6 +16,7 @@ package google.registry.model.transfer;
 
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
+import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.persistence.MappedSuperclass;
@@ -38,6 +39,7 @@ public abstract class BaseTransferObject extends ImmutableObject {
 
   /** The gaining registrar of the current or last transfer. Can be null if never transferred. */
   @XmlElement(name = "reID")
+  @Column(name = "transfer_gaining_registrar_id")
   String gainingClientId;
 
   /** The time that the last transfer was requested. Can be null if never transferred. */
@@ -46,6 +48,7 @@ public abstract class BaseTransferObject extends ImmutableObject {
 
   /** The losing registrar of the current or last transfer. Can be null if never transferred. */
   @XmlElement(name = "acID")
+  @Column(name = "transfer_losing_registrar_id")
   String losingClientId;
 
   /**
@@ -54,6 +57,7 @@ public abstract class BaseTransferObject extends ImmutableObject {
    * this holds the time that the last pending transfer ended. Can be null if never transferred.
    */
   @XmlElement(name = "acDate")
+  @Column(name = "transfer_pending_expiration_time")
   DateTime pendingTransferExpirationTime;
 
   public TransferStatus getTransferStatus() {

core/src/main/java/google/registry/model/transfer/TransferData.java

@@ -17,8 +17,8 @@ package google.registry.model.transfer;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Unindex;
 import com.googlecode.objectify.condition.IfNull;
@@ -29,10 +29,14 @@ import google.registry.model.domain.Period;
 import google.registry.model.domain.Period.Unit;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.poll.PollMessage;
+import google.registry.persistence.VKey;
 import java.util.Set;
 import javax.annotation.Nullable;
-import javax.persistence.ElementCollection;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
 import javax.persistence.Embedded;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -47,7 +51,16 @@ public class TransferData extends BaseTransferObject implements Buildable {
   public static final TransferData EMPTY = new TransferData();
 
   /** The transaction id of the most recent transfer request (or null if there never was one). */
-  @Embedded Trid transferRequestTrid;
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "serverTransactionId",
+        column = @Column(name = "transfer_server_txn_id")),
+    @AttributeOverride(
+        name = "clientTransactionId",
+        column = @Column(name = "transfer_client_txn_id"))
+  })
+  Trid transferRequestTrid;
 
   /**
    * The period to extend the registration upon completion of the transfer.
@@ -55,23 +68,29 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * <p>By default, domain transfers are for one year. This can be changed to zero by using the
    * superuser EPP extension.
    */
-  @Embedded Period transferPeriod = Period.create(1, Unit.YEARS);
+  @Embedded
+  @AttributeOverrides({
+    @AttributeOverride(name = "unit", column = @Column(name = "transfer_renew_period_unit")),
+    @AttributeOverride(name = "value", column = @Column(name = "transfer_renew_period_value"))
+  })
+  Period transferPeriod = Period.create(1, Unit.YEARS);
 
   /**
    * The registration expiration time resulting from the approval - speculative or actual - of the
    * most recent transfer request, applicable for domains only.
    *
    * <p>For pending transfers, this is the expiration time that will take effect under a projected
-   * server approval. For approved transfers, this is the actual expiration time of the domain as
-   * of the moment of transfer completion. For rejected or cancelled transfers, this field will be
+   * server approval. For approved transfers, this is the actual expiration time of the domain as of
+   * the moment of transfer completion. For rejected or cancelled transfers, this field will be
    * reset to null.
    *
    * <p>Note that even when this field is set, it does not necessarily mean that the post-transfer
-   * domain has a new expiration time.  Superuser transfers may not include a bundled 1 year renewal
+   * domain has a new expiration time. Superuser transfers may not include a bundled 1 year renewal
    * at all, or even when a renewal is bundled, for a transfer during the autorenew grace period the
    * bundled renewal simply subsumes the recent autorenewal, resulting in the same expiration time.
    */
   // TODO(b/36405140): backfill this field for existing domains to which it should apply.
+  @Column(name = "transfer_registration_expiration_time")
   DateTime transferredRegistrationExpirationTime;
 
   /**
@@ -83,18 +102,35 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * pending transfer is explicitly approved, rejected or cancelled, the referenced entities should
    * be deleted.
    */
+  @Transient
   @IgnoreSave(IfNull.class)
-  @ElementCollection
-  Set<Key<? extends TransferServerApproveEntity>> serverApproveEntities;
+  Set<VKey<? extends TransferServerApproveEntity>> serverApproveEntities;
+
+  // The following 3 fields are the replacement for serverApproveEntities in Cloud SQL.
+  // TODO(shicong): Add getter/setter for these 3 fields and use them in the application code.
+  @Ignore
+  @Column(name = "transfer_gaining_poll_message_id")
+  Long gainingTransferPollMessageId;
+
+  @Ignore
+  @Column(name = "transfer_losing_poll_message_id")
+  Long losingTransferPollMessageId;
+
+  @Ignore
+  @Column(name = "transfer_billing_cancellation_id")
+  Long billingCancellationId;
 
   /**
    * The regular one-time billing event that will be charged for a server-approved transfer.
    *
    * <p>This field should be null if there is not currently a pending transfer or if the object
    * being transferred is not a domain.
+   *
+   * <p>TODO(b/158230654) Remove unused columns for TransferData in Contact table.
    */
   @IgnoreSave(IfNull.class)
-  Key<BillingEvent.OneTime> serverApproveBillingEvent;
+  @Column(name = "transfer_billing_event_id")
+  VKey<BillingEvent.OneTime> serverApproveBillingEvent;
 
   /**
    * The autorenew billing event that should be associated with this resource after the transfer.
@@ -103,7 +139,8 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * being transferred is not a domain.
    */
   @IgnoreSave(IfNull.class)
-  Key<BillingEvent.Recurring> serverApproveAutorenewEvent;
+  @Column(name = "transfer_billing_recurrence_id")
+  VKey<BillingEvent.Recurring> serverApproveAutorenewEvent;
 
   /**
    * The autorenew poll message that should be associated with this resource after the transfer.
@@ -112,7 +149,8 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * being transferred is not a domain.
    */
   @IgnoreSave(IfNull.class)
-  Key<PollMessage.Autorenew> serverApproveAutorenewPollMessage;
+  @Column(name = "transfer_autorenew_poll_message_id")
+  VKey<PollMessage.Autorenew> serverApproveAutorenewPollMessage;
 
   @Nullable
   public Trid getTransferRequestTrid() {
@@ -128,22 +166,22 @@ public class TransferData extends BaseTransferObject implements Buildable {
     return transferredRegistrationExpirationTime;
   }
 
-  public ImmutableSet<Key<? extends TransferServerApproveEntity>> getServerApproveEntities() {
+  public ImmutableSet<VKey<? extends TransferServerApproveEntity>> getServerApproveEntities() {
     return nullToEmptyImmutableCopy(serverApproveEntities);
   }
 
   @Nullable
-  public Key<BillingEvent.OneTime> getServerApproveBillingEvent() {
+  public VKey<BillingEvent.OneTime> getServerApproveBillingEvent() {
     return serverApproveBillingEvent;
   }
 
   @Nullable
-  public Key<BillingEvent.Recurring> getServerApproveAutorenewEvent() {
+  public VKey<BillingEvent.Recurring> getServerApproveAutorenewEvent() {
     return serverApproveAutorenewEvent;
   }
 
   @Nullable
-  public Key<PollMessage.Autorenew> getServerApproveAutorenewPollMessage() {
+  public VKey<PollMessage.Autorenew> getServerApproveAutorenewPollMessage() {
     return serverApproveAutorenewPollMessage;
   }
 
@@ -202,25 +240,25 @@ public class TransferData extends BaseTransferObject implements Buildable {
     }
 
     public Builder setServerApproveEntities(
-        ImmutableSet<Key<? extends TransferServerApproveEntity>> serverApproveEntities) {
+        ImmutableSet<VKey<? extends TransferServerApproveEntity>> serverApproveEntities) {
       getInstance().serverApproveEntities = serverApproveEntities;
       return this;
     }
 
     public Builder setServerApproveBillingEvent(
-        Key<BillingEvent.OneTime> serverApproveBillingEvent) {
+        VKey<BillingEvent.OneTime> serverApproveBillingEvent) {
       getInstance().serverApproveBillingEvent = serverApproveBillingEvent;
       return this;
     }
 
     public Builder setServerApproveAutorenewEvent(
-        Key<BillingEvent.Recurring> serverApproveAutorenewEvent) {
+        VKey<BillingEvent.Recurring> serverApproveAutorenewEvent) {
       getInstance().serverApproveAutorenewEvent = serverApproveAutorenewEvent;
       return this;
     }
 
     public Builder setServerApproveAutorenewPollMessage(
-        Key<PollMessage.Autorenew> serverApproveAutorenewPollMessage) {
+        VKey<PollMessage.Autorenew> serverApproveAutorenewPollMessage) {
       getInstance().serverApproveAutorenewPollMessage = serverApproveAutorenewPollMessage;
       return this;
     }
@@ -230,5 +268,7 @@ public class TransferData extends BaseTransferObject implements Buildable {
    * Marker interface for objects that are written in anticipation of a server approval, and
    * therefore need to be deleted under any other outcome.
    */
-  public interface TransferServerApproveEntity {}
+  public interface TransferServerApproveEntity {
+    VKey<? extends TransferServerApproveEntity> createVKey();
+  }
 }

core/src/main/java/google/registry/persistence/VKey.java

@@ -17,6 +17,7 @@ package google.registry.persistence;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.googlecode.objectify.Key;
 import google.registry.model.ImmutableObject;
 import java.util.Optional;
 
@@ -57,6 +58,23 @@ public class VKey<T> extends ImmutableObject {
     return new VKey(kind, ofyKey, null);
   }
 
+  /**
+   * Creates a {@link VKey} which only contains the ofy primary key by specifying the id of the
+   * {@link Key}.
+   */
+  public static <T> VKey<T> createOfy(Class<? extends T> kind, long id) {
+    return createOfy(kind, Key.create(kind, id));
+  }
+
+  /**
+   * Creates a {@link VKey} which only contains the ofy primary key by specifying the name of the
+   * {@link Key}.
+   */
+  public static <T> VKey<T> createOfy(Class<? extends T> kind, String name) {
+    checkArgumentNotNull(kind, "name must not be null");
+    return createOfy(kind, Key.create(kind, name));
+  }
+
   /** Creates a {@link VKey} which only contains both sql and ofy primary key. */
   public static <T> VKey<T> create(
       Class<? extends T> kind, Object sqlKey, com.googlecode.objectify.Key ofyKey) {

core/src/main/java/google/registry/persistence/converter/InetAddressSetConverter.java

@@ -0,0 +1,33 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import com.google.common.net.InetAddresses;
+import java.net.InetAddress;
+import javax.persistence.Converter;
+
+@Converter(autoApply = true)
+public class InetAddressSetConverter extends StringSetConverterBase<InetAddress> {
+
+  @Override
+  String toString(InetAddress element) {
+    return InetAddresses.toAddrString(element);
+  }
+
+  @Override
+  InetAddress fromString(String value) {
+    return InetAddresses.forString(value);
+  }
+}

core/src/main/java/google/registry/persistence/converter/TransferServerApproveEntitySetConverter.java

@@ -0,0 +1,37 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
+import google.registry.persistence.VKey;
+import java.util.Set;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** {@link AttributeConverter} for {@link Set}. */
+@Converter(autoApply = true)
+public class TransferServerApproveEntitySetConverter
+    extends StringSetConverterBase<VKey<? extends TransferServerApproveEntity>> {
+
+  @Override
+  String toString(VKey<? extends TransferServerApproveEntity> element) {
+    return String.valueOf(element.getSqlKey());
+  }
+
+  @Override
+  VKey<? extends TransferServerApproveEntity> fromString(String value) {
+    return VKey.createSql(TransferServerApproveEntity.class, Long.parseLong(value));
+  }
+}

core/src/main/java/google/registry/rdap/RdapJsonFormatter.java

@@ -433,10 +433,7 @@ public class RdapJsonFormatter {
         statuses.add(StatusValue.LINKED);
       }
       if (hostResource.isSubordinate()
-          && ofy()
-              .load()
-              .key(hostResource.getSuperordinateDomain())
-              .now()
+          && tm().load(hostResource.getSuperordinateDomain())
               .cloneProjectedAtTime(getRequestTime())
               .getStatusValues()
               .contains(StatusValue.PENDING_TRANSFER)) {

core/src/main/java/google/registry/rde/HostResourceToXjcConverter.java

@@ -17,7 +17,6 @@ package google.registry.rde;
 import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.net.InetAddresses;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
@@ -35,9 +34,8 @@ import org.joda.time.DateTime;
 final class HostResourceToXjcConverter {
 
   /** Converts a subordinate {@link HostResource} to {@link XjcRdeHostElement}. */
-  static XjcRdeHostElement convertSubordinate(
-      HostResource host, DomainBase superordinateDomain) {
-    checkArgument(Key.create(superordinateDomain).equals(host.getSuperordinateDomain()));
+  static XjcRdeHostElement convertSubordinate(HostResource host, DomainBase superordinateDomain) {
+    checkArgument(superordinateDomain.createVKey().equals(host.getSuperordinateDomain()));
     return new XjcRdeHostElement(convertSubordinateHost(host, superordinateDomain));
   }
 

core/src/main/java/google/registry/rde/RdeStagingMapper.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Strings.nullToEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.EppResourceUtils.loadAtPointInTime;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.appengine.tools.mapreduce.Mapper;
 import com.google.auto.value.AutoValue;
@@ -186,13 +187,16 @@ public final class RdeStagingMapper extends Mapper<EppResource, PendingDeposit,
         return result;
       } else if (resource instanceof HostResource) {
         HostResource host = (HostResource) resource;
-        result = Optional.of(host.isSubordinate()
-            ? marshaller.marshalSubordinateHost(
-                host,
-                // Note that loadAtPointInTime() does cloneProjectedAtTime(watermark) for us.
-                loadAtPointInTime(
-                    ofy().load().key(host.getSuperordinateDomain()).now(), watermark).now())
-            : marshaller.marshalExternalHost(host));
+        result =
+            Optional.of(
+                host.isSubordinate()
+                    ? marshaller.marshalSubordinateHost(
+                        host,
+                        // Note that loadAtPointInTime() does cloneProjectedAtTime(watermark) for
+                        // us.
+                        loadAtPointInTime(tm().load(host.getSuperordinateDomain()), watermark)
+                            .now())
+                    : marshaller.marshalExternalHost(host));
         cache.put(WatermarkModePair.create(watermark, RdeMode.FULL), result);
         cache.put(WatermarkModePair.create(watermark, RdeMode.THIN), result);
         return result;

core/src/main/java/google/registry/tools/LevelDbLogReader.java

@@ -88,8 +88,8 @@ public final class LevelDbLogReader implements Iterator<byte[]> {
   }
 
   /**
-   * Returns the next {@link #BLOCK_SIZE} bytes from the input channel, or {@link
-   * Optional#empty()} if there is no more data.
+   * Returns the next {@link #BLOCK_SIZE} bytes from the input channel, or {@link Optional#empty()}
+   * if there is no more data.
    */
   // TODO(weiminyu): use ByteBuffer directly.
   private Optional<byte[]> readFromChannel() throws IOException {

core/src/main/java/google/registry/ui/server/registrar/RegistryLockPostAction.java

@@ -30,7 +30,6 @@ import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.gson.Gson;
-import google.registry.config.RegistryConfig;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
@@ -49,12 +48,12 @@ import google.registry.tools.DomainLockUtils;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
 import java.net.URISyntaxException;
-import java.net.URL;
 import java.util.Map;
 import java.util.Optional;
 import javax.inject.Inject;
 import javax.mail.internet.AddressException;
 import javax.mail.internet.InternetAddress;
+import javax.servlet.http.HttpServletRequest;
 import org.apache.http.client.utils.URIBuilder;
 import org.joda.time.Duration;
 
@@ -76,11 +75,11 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final Gson GSON = new Gson();
 
-  private static final URL URL_BASE = RegistryConfig.getDefaultServer();
   private static final String VERIFICATION_EMAIL_TEMPLATE =
       "Please click the link below to perform the lock / unlock action on domain %s. Note: "
           + "this code will expire in one hour.\n\n%s";
 
+  private final HttpServletRequest req;
   private final JsonActionRunner jsonActionRunner;
   private final AuthResult authResult;
   private final AuthenticatedRegistrarAccessor registrarAccessor;
@@ -90,12 +89,14 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
 
   @Inject
   RegistryLockPostAction(
+      HttpServletRequest req,
       JsonActionRunner jsonActionRunner,
       AuthResult authResult,
       AuthenticatedRegistrarAccessor registrarAccessor,
       SendEmailService sendEmailService,
       DomainLockUtils domainLockUtils,
       @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress) {
+    this.req = req;
     this.jsonActionRunner = jsonActionRunner;
     this.authResult = authResult;
     this.registrarAccessor = registrarAccessor;
@@ -161,7 +162,7 @@ public class RegistryLockPostAction implements Runnable, JsonActionRunner.JsonAc
       String url =
           new URIBuilder()
               .setScheme("https")
-              .setHost(URL_BASE.getHost())
+              .setHost(req.getServerName())
               .setPath("registry-lock-verify")
               .setParameter("lockVerificationCode", lock.getVerificationCode())
               .setParameter("isLock", String.valueOf(isLock))

core/src/main/java/google/registry/whois/NameserverWhoisResponse.java

@@ -16,7 +16,7 @@ package google.registry.whois;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.net.InetAddresses;
@@ -49,7 +49,7 @@ final class NameserverWhoisResponse extends WhoisResponseImpl {
       HostResource host = hosts.get(i);
       String clientId =
           host.isSubordinate()
-              ? ofy().load().key(host.getSuperordinateDomain()).now()
+              ? tm().load(host.getSuperordinateDomain())
                   .cloneProjectedAtTime(getTimestamp())
                   .getCurrentSponsorClientId()
               : host.getPersistedCurrentSponsorClientId();

core/src/main/resources/META-INF/persistence.xml

@@ -50,20 +50,25 @@
     <class>google.registry.persistence.converter.CurrencyUnitConverter</class>
     <class>google.registry.persistence.converter.DateTimeConverter</class>
     <class>google.registry.persistence.converter.DurationConverter</class>
+    <class>google.registry.persistence.converter.InetAddressSetConverter</class>
     <class>google.registry.persistence.converter.PostalInfoChoiceListConverter</class>
     <class>google.registry.persistence.converter.RegistrarPocSetConverter</class>
     <class>google.registry.persistence.converter.StatusValueSetConverter</class>
     <class>google.registry.persistence.converter.StringListConverter</class>
     <class>google.registry.persistence.converter.StringSetConverter</class>
     <class>google.registry.persistence.converter.TldStateTransitionConverter</class>
+    <class>google.registry.persistence.converter.TransferServerApproveEntitySetConverter</class>
     <class>google.registry.persistence.converter.UpdateAutoTimestampConverter</class>
     <class>google.registry.persistence.converter.ZonedDateTimeConverter</class>
 
     <!-- Generated converters for VKey -->
     <class>google.registry.model.billing.VKeyConverter_BillingEvent</class>
+    <class>google.registry.model.domain.VKeyConverter_DomainBase</class>
+    <class>google.registry.model.contact.VKeyConverter_ContactResource</class>
     <class>google.registry.model.domain.token.VKeyConverter_AllocationToken</class>
     <class>google.registry.model.host.VKeyConverter_HostResource</class>
-    <class>google.registry.model.contact.VKeyConverter_ContactResource</class>
+    <class>google.registry.model.poll.VKeyConverter_Autorenew</class>
+    <class>google.registry.model.poll.VKeyConverter_OneTime</class>
 
     <!-- TODO(weiminyu): check out application-layer validation. -->
     <validation-mode>NONE</validation-mode>

core/src/test/java/google/registry/backup/CommitLogExports.java

@@ -0,0 +1,231 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Lists.partition;
+import static google.registry.backup.BackupUtils.serializeEntity;
+import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static google.registry.util.DateTimeUtils.isAtOrAfter;
+import static java.util.Comparator.comparingLong;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Streams;
+import com.googlecode.objectify.Key;
+import google.registry.model.ImmutableObject;
+import google.registry.model.ofy.CommitLogBucket;
+import google.registry.model.ofy.CommitLogCheckpoint;
+import google.registry.model.ofy.CommitLogCheckpointRoot;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import google.registry.util.Clock;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import org.joda.time.DateTime;
+
+/**
+ * Helpers for exporting the diff between two commit log checkpoints to a local file.
+ *
+ * <p>In production, CommitLogs are saved periodically by cron jobs. During each job, the {@link
+ * CommitLogCheckpointAction} is invoked first to compute a {@link CommitLogCheckpoint} and persist
+ * it in Datastore. Then the {@link ExportCommitLogDiffAction} is invoked to export the diffs
+ * accumulated between the previous and current checkpoints to a file.
+ *
+ * <p>The {@link #computeCheckpoint(Clock)} method is copied with simplification from {@link
+ * CommitLogCheckpointAction}, and the {@link #saveCommitLogs(String, CommitLogCheckpoint,
+ * CommitLogCheckpoint)} method is copied with simplification from {@link
+ * ExportCommitLogDiffAction}. We opted for copying instead of refactoring to reduce risk to
+ * production code.
+ */
+public final class CommitLogExports {
+
+  public static final String DIFF_FILE_PREFIX = "commit_diff_until_";
+
+  private static final int EXPORT_DIFF_BATCH_SIZE = 100;
+
+  private CommitLogExports() {}
+
+  /**
+   * Returns the next {@link CommitLogCheckpoint} for Commit logs. Please refer to the class javadoc
+   * for background.
+   */
+  public static CommitLogCheckpoint computeCheckpoint(Clock clock) {
+    CommitLogCheckpointStrategy strategy = new CommitLogCheckpointStrategy();
+    strategy.clock = clock;
+    strategy.ofy = ofy();
+
+    CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
+    tm().transact(
+            () -> {
+              DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
+              checkState(
+                  checkpoint.getCheckpointTime().isAfter(lastWrittenTime),
+                  "Newer checkpoint already written at time: %s",
+                  lastWrittenTime);
+              ofy()
+                  .saveWithoutBackup()
+                  .entities(
+                      checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
+            });
+    return checkpoint;
+  }
+
+  /**
+   * Saves the incremental changes between {@code prevCheckpoint} and {@code checkpoint} and returns
+   * the {@link File}. Please refer to class javadoc for background.
+   */
+  public static File saveCommitLogs(
+      String commitLogDir,
+      @Nullable CommitLogCheckpoint prevCheckpoint,
+      CommitLogCheckpoint checkpoint) {
+    checkArgument(
+        prevCheckpoint == null
+            || (isAtOrAfter(prevCheckpoint.getCheckpointTime(), START_OF_TIME)
+                && prevCheckpoint.getCheckpointTime().isBefore(checkpoint.getCheckpointTime())),
+        "Inversed checkpoint: prev is %s, current is %s.",
+        Optional.ofNullable(prevCheckpoint)
+            .map(CommitLogCheckpoint::getCheckpointTime)
+            .map(DateTime::toString)
+            .orElse("null"),
+        checkpoint.getCheckpointTime().toString());
+
+    // Load the keys of all the manifests to include in this diff.
+    List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(prevCheckpoint, checkpoint);
+    // Open an output channel to GCS, wrapped in a stream for convenience.
+    File commitLogFile =
+        new File(commitLogDir + "/" + DIFF_FILE_PREFIX + checkpoint.getCheckpointTime());
+    try (OutputStream commitLogStream =
+        new BufferedOutputStream(new FileOutputStream(commitLogFile))) {
+      // Export the upper checkpoint itself.
+      serializeEntity(checkpoint, commitLogStream);
+      // If there are no manifests to export, stop early, now that we've written out the file with
+      // the checkpoint itself (which is needed for restores, even if it's empty).
+      if (sortedKeys.isEmpty()) {
+        return commitLogFile;
+      }
+      // Export to GCS in chunks, one per fixed batch of commit logs.  While processing one batch,
+      // asynchronously load the entities for the next one.
+      List<List<Key<CommitLogManifest>>> keyChunks = partition(sortedKeys, EXPORT_DIFF_BATCH_SIZE);
+      // Objectify's map return type is asynchronous. Calling .values() will block until it loads.
+      Map<?, CommitLogManifest> nextChunkToExport = ofy().load().keys(keyChunks.get(0));
+      for (int i = 0; i < keyChunks.size(); i++) {
+        // Force the async load to finish.
+        Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
+        // Since there is no hard bound on how much data this might be, take care not to let the
+        // Objectify session cache fill up and potentially run out of memory. This is the only safe
+        // point to do this since at this point there is no async load in progress.
+        ofy().clearSessionCache();
+        // Kick off the next async load, which can happen in parallel to the current GCS export.
+        if (i + 1 < keyChunks.size()) {
+          nextChunkToExport = ofy().load().keys(keyChunks.get(i + 1));
+        }
+        exportChunk(commitLogStream, chunkValues);
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return commitLogFile;
+  }
+
+  /**
+   * Loads all the diff keys, sorted in a transaction-consistent chronological order.
+   *
+   * @param lowerCheckpoint exclusive lower bound on keys in this diff, or null if no lower bound
+   * @param upperCheckpoint inclusive upper bound on keys in this diff
+   */
+  private static ImmutableList<Key<CommitLogManifest>> loadAllDiffKeys(
+      @Nullable final CommitLogCheckpoint lowerCheckpoint,
+      final CommitLogCheckpoint upperCheckpoint) {
+    // Fetch the keys (no data) between these checkpoints, and sort by timestamp. This ordering is
+    // transaction-consistent by virtue of our checkpoint strategy and our customized Ofy; see
+    // CommitLogCheckpointStrategy for the proof. We break ties by sorting on bucket ID to ensure
+    // a deterministic order.
+    return upperCheckpoint.getBucketTimestamps().keySet().stream()
+        .flatMap(
+            bucketNum ->
+                Streams.stream(loadDiffKeysFromBucket(lowerCheckpoint, upperCheckpoint, bucketNum)))
+        .sorted(
+            comparingLong(Key<CommitLogManifest>::getId)
+                .thenComparingLong(a -> a.getParent().getId()))
+        .collect(toImmutableList());
+  }
+
+  /**
+   * Loads the diff keys for one bucket.
+   *
+   * @param lowerCheckpoint exclusive lower bound on keys in this diff, or null if no lower bound
+   * @param upperCheckpoint inclusive upper bound on keys in this diff
+   * @param bucketNum the bucket to load diff keys from
+   */
+  private static Iterable<Key<CommitLogManifest>> loadDiffKeysFromBucket(
+      @Nullable CommitLogCheckpoint lowerCheckpoint,
+      CommitLogCheckpoint upperCheckpoint,
+      int bucketNum) {
+    // If no lower checkpoint exists, or if it exists but had no timestamp for this bucket number
+    // (because the bucket count was increased between these checkpoints), then use START_OF_TIME
+    // as the effective exclusive lower bound.
+    DateTime lowerCheckpointBucketTime =
+        firstNonNull(
+            (lowerCheckpoint == null) ? null : lowerCheckpoint.getBucketTimestamps().get(bucketNum),
+            START_OF_TIME);
+    // Since START_OF_TIME=0 is not a valid id in a key, add 1 to both bounds. Then instead of
+    // loading lowerBound < x <= upperBound, we can load lowerBound <= x < upperBound.
+    DateTime lowerBound = lowerCheckpointBucketTime.plusMillis(1);
+    DateTime upperBound = upperCheckpoint.getBucketTimestamps().get(bucketNum).plusMillis(1);
+    // If the lower and upper bounds are equal, there can't be any results, so skip the query.
+    if (lowerBound.equals(upperBound)) {
+      return ImmutableSet.of();
+    }
+    Key<CommitLogBucket> bucketKey = getBucketKey(bucketNum);
+    return ofy()
+        .load()
+        .type(CommitLogManifest.class)
+        .ancestor(bucketKey)
+        .filterKey(">=", CommitLogManifest.createKey(bucketKey, lowerBound))
+        .filterKey("<", CommitLogManifest.createKey(bucketKey, upperBound))
+        .keys();
+  }
+
+  /** Writes a chunks-worth of manifests and associated mutations to GCS. */
+  private static void exportChunk(OutputStream gcsStream, Collection<CommitLogManifest> chunk)
+      throws IOException {
+    // Kickoff async loads for all the manifests in the chunk.
+    ImmutableList.Builder<Iterable<? extends ImmutableObject>> entities =
+        new ImmutableList.Builder<>();
+    for (CommitLogManifest manifest : chunk) {
+      entities.add(ImmutableList.of(manifest));
+      entities.add(ofy().load().type(CommitLogMutation.class).ancestor(manifest));
+    }
+    for (ImmutableObject entity : concat(entities.build())) {
+      serializeEntity(entity, gcsStream);
+    }
+  }
+}

core/src/test/java/google/registry/batch/DeleteContactsAndHostsActionTest.java

@@ -725,7 +725,7 @@ public class DeleteContactsAndHostsActionTest
         persistResource(
             persistHostPendingDelete("ns2.example.tld")
                 .asBuilder()
-                .setSuperordinateDomain(Key.create(domain))
+                .setSuperordinateDomain(domain.createVKey())
                 .build());
     enqueuer.enqueueAsyncDelete(
         host,

core/src/test/java/google/registry/beam/initsql/BackupPathsTest.java

@@ -1,50 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.initsql;
-
-import static com.google.common.truth.Truth.assertThat;
-import static google.registry.beam.initsql.BackupPaths.getKindFromFileName;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
-import org.junit.jupiter.api.Test;
-
-/** Unit tests for {@link google.registry.beam.initsql.BackupPaths}. */
-public class BackupPathsTest {
-
-  @Test
-  void getKindFromFileName_empty() {
-    assertThrows(IllegalArgumentException.class, () -> getKindFromFileName(""));
-  }
-
-  @Test
-  void getKindFromFileName_notMatch() {
-    assertThrows(
-        IllegalArgumentException.class,
-        () -> getKindFromFileName("/tmp/all_namespaces/kind_/input-0"));
-  }
-
-  @Test
-  void getKindFromFileName_success() {
-    assertThat(getKindFromFileName("scheme:/somepath/all_namespaces/kind_mykind/input-something"))
-        .isEqualTo("mykind");
-  }
-
-  @Test
-  void getKindFromFileName_specialChar_success() {
-    assertThat(
-            getKindFromFileName("scheme:/somepath/all_namespaces/kind_.*+? /(a)/input-something"))
-        .isEqualTo(".*+? /(a)");
-  }
-}

core/src/test/java/google/registry/beam/initsql/BackupTestStore.java

@@ -20,6 +20,8 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 
 import com.google.appengine.api.datastore.Entity;
 import com.googlecode.objectify.Key;
+import google.registry.backup.CommitLogExports;
+import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.testing.AppEngineRule;
 import google.registry.testing.FakeClock;
 import google.registry.tools.LevelDbFileBuilder;
@@ -35,6 +37,10 @@ import org.joda.time.format.DateTimeFormatter;
  * <p>A Datastore backup consists of an unsynchronized data export and a sequence of incremental
  * Commit Logs that overlap with the export process. Together they can be used to recreate a
  * consistent snapshot of the Datastore.
+ *
+ * <p>For convenience of test-writing, the {@link #fakeClock} is advanced by 1 millisecond after
+ * every transaction is invoked on this store, ensuring strictly increasing timestamps on causally
+ * dependent transactions. In production, the same ordering is ensured by sleep and retry.
  */
 class BackupTestStore implements AutoCloseable {
 
@@ -44,6 +50,8 @@ class BackupTestStore implements AutoCloseable {
   private final FakeClock fakeClock;
   private AppEngineRule appEngine;
 
+  private CommitLogCheckpoint prevCommitLogCheckpoint;
+
   BackupTestStore(FakeClock fakeClock) throws Exception {
     this.fakeClock = fakeClock;
     this.appEngine =
@@ -55,16 +63,27 @@ class BackupTestStore implements AutoCloseable {
     this.appEngine.beforeEach(null);
   }
 
+  void transact(Iterable<Object> deletes, Iterable<Object> newOrUpdated) {
+    tm().transact(
+            () -> {
+              ofy().delete().entities(deletes);
+              ofy().save().entities(newOrUpdated);
+            });
+    fakeClock.advanceOneMilli();
+  }
+
   /** Inserts or updates {@code entities} in the Datastore. */
   @SafeVarargs
   final void insertOrUpdate(Object... entities) {
     tm().transact(() -> ofy().save().entities(entities).now());
+    fakeClock.advanceOneMilli();
   }
 
   /** Deletes {@code entities} from the Datastore. */
   @SafeVarargs
   final void delete(Object... entities) {
     tm().transact(() -> ofy().delete().entities(entities).now());
+    fakeClock.advanceOneMilli();
   }
 
   /**
@@ -109,8 +128,12 @@ class BackupTestStore implements AutoCloseable {
     builder.build();
   }
 
-  void saveCommitLog() {
-    throw new UnsupportedOperationException("Not implemented yet");
+  File saveCommitLogs(String commitLogDir) {
+    CommitLogCheckpoint checkpoint = CommitLogExports.computeCheckpoint(fakeClock);
+    File commitLogFile =
+        CommitLogExports.saveCommitLogs(commitLogDir, prevCommitLogCheckpoint, checkpoint);
+    prevCommitLogCheckpoint = checkpoint;
+    return commitLogFile;
   }
 
   @Override

core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java

@@ -19,27 +19,36 @@ import static com.google.common.truth.Truth.assertWithMessage;
 import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatastoreHelper.newContactResource;
 import static google.registry.testing.DatastoreHelper.newDomainBase;
 import static google.registry.testing.DatastoreHelper.newRegistry;
 
 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
 import com.googlecode.objectify.Key;
+import google.registry.backup.CommitLogImports;
+import google.registry.backup.VersionedEntity;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DesignatedContact;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.Ofy;
 import google.registry.model.registry.Registry;
+import google.registry.persistence.VKey;
 import google.registry.testing.FakeClock;
+import google.registry.testing.InjectRule;
 import google.registry.tools.LevelDbLogReader;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -47,6 +56,7 @@ import org.joda.time.DateTime;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 import org.junit.jupiter.api.io.TempDir;
 
 /** Unit tests for {@link BackupTestStore}. */
@@ -56,17 +66,25 @@ public class BackupTestStoreTest {
   private FakeClock fakeClock;
   private BackupTestStore store;
 
+  private Registry registry;
+  private ContactResource contact;
+  private DomainBase domain;
+
   @TempDir File tempDir;
 
+  @RegisterExtension InjectRule injectRule = new InjectRule();
+
   @BeforeEach
   void beforeEach() throws Exception {
     fakeClock = new FakeClock(START_TIME);
     store = new BackupTestStore(fakeClock);
+    injectRule.setStaticField(Ofy.class, "clock", fakeClock);
 
-    store.insertOrUpdate(newRegistry("tld1", "TLD1"));
-    ContactResource contact1 = newContactResource("contact_1");
-    DomainBase domain1 = newDomainBase("domain1.tld1", contact1);
-    store.insertOrUpdate(contact1, domain1);
+    registry = newRegistry("tld1", "TLD1");
+    store.insertOrUpdate(registry);
+    contact = newContactResource("contact_1");
+    domain = newDomainBase("domain1.tld1", contact);
+    store.insertOrUpdate(contact, domain);
   }
 
   @AfterEach
@@ -77,7 +95,8 @@ public class BackupTestStoreTest {
   @Test
   void export_filesCreated() throws IOException {
     String exportRootPath = tempDir.getAbsolutePath();
-    File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_000");
+    assertThat(fakeClock.nowUtc().toString()).isEqualTo("2000-01-01T00:00:00.002Z");
+    File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_002");
     assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile())
         .that(exportFolder.exists())
         .isFalse();
@@ -100,7 +119,7 @@ public class BackupTestStoreTest {
   void export_folderNameChangesWithTime() throws IOException {
     String exportRootPath = tempDir.getAbsolutePath();
     fakeClock.advanceOneMilli();
-    File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_001");
+    File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_003");
     assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile())
         .that(exportFolder.exists())
         .isFalse();
@@ -147,6 +166,69 @@ public class BackupTestStoreTest {
     assertThat(tlds).containsExactly("tld2");
   }
 
+  @Test
+  void saveCommitLogs_fileCreated() {
+    File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath());
+    assertThat(commitLogFile.exists()).isTrue();
+    assertThat(commitLogFile.getName()).isEqualTo("commit_diff_until_2000-01-01T00:00:00.002Z");
+  }
+
+  @Test
+  void saveCommitLogs_inserts() {
+    File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath());
+    assertThat(commitLogFile.exists()).isTrue();
+    ImmutableList<VersionedEntity> mutations = CommitLogImports.loadEntities(commitLogFile);
+    assertThat(mutations.stream().map(VersionedEntity::getEntity).map(Optional::get))
+        .containsExactlyElementsIn(toDatastoreEntities(registry, contact, domain));
+    // Registry created at -2, contract and domain created at -1.
+    assertThat(mutations.stream().map(VersionedEntity::commitTimeMills))
+        .containsExactly(
+            fakeClock.nowUtc().getMillis() - 2,
+            fakeClock.nowUtc().getMillis() - 1,
+            fakeClock.nowUtc().getMillis() - 1);
+  }
+
+  @Test
+  void saveCommitLogs_deletes() {
+    fakeClock.advanceOneMilli();
+    store.saveCommitLogs(tempDir.getAbsolutePath());
+    ContactResource newContact = newContactResource("contact2");
+    VKey<ContactResource> vKey = newContact.createVKey();
+    domain =
+        domain
+            .asBuilder()
+            .setRegistrant(vKey)
+            .setContacts(
+                ImmutableSet.of(
+                    DesignatedContact.create(DesignatedContact.Type.ADMIN, vKey),
+                    DesignatedContact.create(DesignatedContact.Type.TECH, vKey)))
+            .build();
+    store.insertOrUpdate(domain, newContact);
+    store.delete(contact);
+    fakeClock.advanceOneMilli();
+    File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath());
+    ImmutableList<VersionedEntity> mutations = CommitLogImports.loadEntities(commitLogFile);
+    assertThat(mutations.stream().filter(VersionedEntity::isDelete).map(VersionedEntity::key))
+        .containsExactly(Key.create(contact).getRaw());
+
+    assertThat(
+            mutations.stream()
+                .filter(Predicates.not(VersionedEntity::isDelete))
+                .map(VersionedEntity::getEntity)
+                .map(Optional::get))
+        .containsExactlyElementsIn(toDatastoreEntities(domain, newContact));
+  }
+
+  @Test
+  void saveCommitLogs_empty() {
+    fakeClock.advanceOneMilli();
+    store.saveCommitLogs(tempDir.getAbsolutePath());
+    fakeClock.advanceOneMilli();
+    File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath());
+    assertThat(commitLogFile.exists()).isTrue();
+    assertThat(CommitLogImports.loadEntities(commitLogFile)).isEmpty();
+  }
+
   private File export(String exportRootPath, Set<Key<?>> excludes) throws IOException {
     return store.export(
         exportRootPath,
@@ -168,4 +250,13 @@ public class BackupTestStoreTest {
     Entity entity = EntityTranslator.createFromPb(proto);
     return ofyEntityType.cast(ofy().load().fromEntity(entity));
   }
+
+  @SafeVarargs
+  private static ImmutableList<Entity> toDatastoreEntities(Object... ofyEntities) {
+    return tm().transact(
+            () ->
+                Stream.of(ofyEntities)
+                    .map(oe -> ofy().save().toEntity(oe))
+                    .collect(ImmutableList.toImmutableList()));
+  }
 }

core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java

@@ -0,0 +1,223 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatastoreHelper.newContactResource;
+import static google.registry.testing.DatastoreHelper.newDomainBase;
+import static google.registry.testing.DatastoreHelper.newRegistry;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.common.collect.ImmutableList;
+import google.registry.backup.VersionedEntity;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.Ofy;
+import google.registry.model.registry.Registry;
+import google.registry.testing.FakeClock;
+import google.registry.testing.InjectRule;
+import java.io.File;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.DateTime;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link CommitLogTransforms}. */
+// TODO(weiminyu): Upgrade to JUnit5 when TestPipeline is upgraded. It is also easy to adapt with
+// a wrapper.
+@RunWith(JUnit4.class)
+public class CommitLogTransformsTest implements Serializable {
+  private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z");
+
+  @Rule public final transient TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  @Rule public final transient InjectRule injectRule = new InjectRule();
+
+  @Rule
+  public final transient TestPipeline pipeline =
+      TestPipeline.create().enableAbandonedNodeEnforcement(true);
+
+  private FakeClock fakeClock;
+  private transient BackupTestStore store;
+  private File commitLogsDir;
+  private File firstCommitLogFile;
+  // Canned data that are persisted to Datastore, used by assertions in tests.
+  // TODO(weiminyu): use Ofy entity pojos directly.
+  private transient ImmutableList<Entity> persistedEntities;
+
+  @Before
+  public void beforeEach() throws Exception {
+    fakeClock = new FakeClock(START_TIME);
+    store = new BackupTestStore(fakeClock);
+    injectRule.setStaticField(Ofy.class, "clock", fakeClock);
+
+    Registry registry = newRegistry("tld1", "TLD1");
+    store.insertOrUpdate(registry);
+    ContactResource contact1 = newContactResource("contact_1");
+    DomainBase domain1 = newDomainBase("domain1.tld1", contact1);
+    store.insertOrUpdate(contact1, domain1);
+    persistedEntities =
+        ImmutableList.of(registry, contact1, domain1).stream()
+            .map(ofyEntity -> tm().transact(() -> ofy().save().toEntity(ofyEntity)))
+            .collect(ImmutableList.toImmutableList());
+    commitLogsDir = temporaryFolder.newFolder();
+    firstCommitLogFile = store.saveCommitLogs(commitLogsDir.getAbsolutePath());
+  }
+
+  @After
+  public void afterEach() throws Exception {
+    if (store != null) {
+      store.close();
+      store = null;
+    }
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void getCommitLogFilePatterns() {
+    PCollection<String> patterns =
+        pipeline.apply(
+            "Get CommitLog file patterns",
+            CommitLogTransforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath()));
+
+    ImmutableList<String> expectedPatterns =
+        ImmutableList.of(commitLogsDir.getAbsolutePath() + "/commit_diff_until_*");
+
+    PAssert.that(patterns).containsInAnyOrder(expectedPatterns);
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void getFilesByPatterns() {
+    PCollection<Metadata> fileMetas =
+        pipeline
+            .apply(
+                "File patterns to metadata",
+                Create.of(commitLogsDir.getAbsolutePath() + "/commit_diff_until_*")
+                    .withCoder(StringUtf8Coder.of()))
+            .apply(Transforms.getFilesByPatterns());
+
+    // Transform fileMetas to file names for assertions.
+    PCollection<String> fileNames =
+        fileMetas.apply(
+            "File metadata to path string",
+            ParDo.of(
+                new DoFn<Metadata, String>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element Metadata metadata, OutputReceiver<String> out) {
+                    out.output(metadata.resourceId().toString());
+                  }
+                }));
+
+    ImmutableList<String> expectedFilenames =
+        ImmutableList.of(firstCommitLogFile.getAbsolutePath());
+
+    PAssert.that(fileNames).containsInAnyOrder(expectedFilenames);
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void filterCommitLogsByTime() {
+    ImmutableList<String> commitLogFilenames =
+        ImmutableList.of(
+            "/commit_diff_until_2000-01-01T00:00:00.000Z",
+            "/commit_diff_until_2000-01-01T00:00:00.001Z",
+            "/commit_diff_until_2000-01-01T00:00:00.002Z",
+            "/commit_diff_until_2000-01-01T00:00:00.003Z",
+            "/commit_diff_until_2000-01-01T00:00:00.004Z");
+    PCollection<String> filteredFilenames =
+        pipeline
+            .apply(
+                "Generate All Filenames",
+                Create.of(commitLogFilenames).withCoder(StringUtf8Coder.of()))
+            .apply(
+                "Filtered by Time",
+                CommitLogTransforms.filterCommitLogsByTime(
+                    DateTime.parse("2000-01-01T00:00:00.001Z"),
+                    DateTime.parse("2000-01-01T00:00:00.003Z")));
+    PAssert.that(filteredFilenames)
+        .containsInAnyOrder(
+            "/commit_diff_until_2000-01-01T00:00:00.001Z",
+            "/commit_diff_until_2000-01-01T00:00:00.002Z");
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void loadOneCommitLogFile() {
+    PCollection<VersionedEntity> entities =
+        pipeline
+            .apply(
+                "Get CommitLog file patterns",
+                CommitLogTransforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath()))
+            .apply("Find CommitLogs", Transforms.getFilesByPatterns())
+            .apply(CommitLogTransforms.loadCommitLogsFromFiles());
+
+    PCollection<Long> timestamps =
+        entities.apply(
+            "Extract commitTimeMillis",
+            ParDo.of(
+                new DoFn<VersionedEntity, Long>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity entity, OutputReceiver<Long> out) {
+                    out.output(entity.commitTimeMills());
+                  }
+                }));
+    PAssert.that(timestamps)
+        .containsInAnyOrder(
+            fakeClock.nowUtc().getMillis() - 2,
+            fakeClock.nowUtc().getMillis() - 1,
+            fakeClock.nowUtc().getMillis() - 1);
+
+    PCollection<Entity> datastoreEntities =
+        entities.apply(
+            "To Datastore Entities",
+            ParDo.of(
+                new DoFn<VersionedEntity, Entity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity entity, OutputReceiver<Entity> out) {
+                    entity.getEntity().ifPresent(out::output);
+                  }
+                }));
+
+    PAssert.that(datastoreEntities).containsInAnyOrder(persistedEntities);
+
+    pipeline.run();
+  }
+}

core/src/test/java/google/registry/beam/initsql/ExportloadingTransformsTest.java

@@ -21,14 +21,15 @@ import static google.registry.testing.DatastoreHelper.newDomainBase;
 import static google.registry.testing.DatastoreHelper.newRegistry;
 
 import com.google.appengine.api.datastore.Entity;
-import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.common.collect.ImmutableList;
-import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
 import com.googlecode.objectify.Key;
+import google.registry.backup.VersionedEntity;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.ofy.Ofy;
 import google.registry.model.registry.Registry;
 import google.registry.testing.FakeClock;
+import google.registry.testing.InjectRule;
 import java.io.File;
 import java.io.Serializable;
 import java.util.Collections;
@@ -40,8 +41,8 @@ import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.DateTime;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -60,6 +61,8 @@ import org.junit.runners.JUnit4;
 // a wrapper.
 @RunWith(JUnit4.class)
 public class ExportloadingTransformsTest implements Serializable {
+  private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z");
+
   private static final ImmutableList<Class<?>> ALL_KINDS =
       ImmutableList.of(Registry.class, ContactResource.class, DomainBase.class);
   private static final ImmutableList<String> ALL_KIND_STRS =
@@ -67,6 +70,8 @@ public class ExportloadingTransformsTest implements Serializable {
 
   @Rule public final transient TemporaryFolder exportRootDir = new TemporaryFolder();
 
+  @Rule public final transient InjectRule injectRule = new InjectRule();
+
   @Rule
   public final transient TestPipeline pipeline =
       TestPipeline.create().enableAbandonedNodeEnforcement(true);
@@ -80,8 +85,9 @@ public class ExportloadingTransformsTest implements Serializable {
 
   @Before
   public void beforeEach() throws Exception {
-    fakeClock = new FakeClock();
+    fakeClock = new FakeClock(START_TIME);
     store = new BackupTestStore(fakeClock);
+    injectRule.setStaticField(Ofy.class, "clock", fakeClock);
 
     Registry registry = newRegistry("tld1", "TLD1");
     store.insertOrUpdate(registry);
@@ -107,7 +113,7 @@ public class ExportloadingTransformsTest implements Serializable {
 
   @Test
   @Category(NeedsRunner.class)
-  public void getBackupDataFilePatterns() {
+  public void getExportFilePatterns() {
     PCollection<String> patterns =
         pipeline.apply(
             "Get Datastore file patterns",
@@ -138,7 +144,7 @@ public class ExportloadingTransformsTest implements Serializable {
                         exportDir.getAbsolutePath()
                             + "/all_namespaces/kind_ContactResource/input-*")
                     .withCoder(StringUtf8Coder.of()))
-            .apply(ExportLoadingTransforms.getFilesByPatterns());
+            .apply(Transforms.getFilesByPatterns());
 
     // Transform fileMetas to file names for assertions.
     PCollection<String> fileNames =
@@ -166,25 +172,25 @@ public class ExportloadingTransformsTest implements Serializable {
 
   @Test
   public void loadDataFromFiles() {
-    PCollection<KV<String, byte[]>> taggedRecords =
+    PCollection<VersionedEntity> taggedRecords =
         pipeline
             .apply(
                 "Get Datastore file patterns",
                 ExportLoadingTransforms.getDatastoreExportFilePatterns(
                     exportDir.getAbsolutePath(), ALL_KIND_STRS))
-            .apply("Find Datastore files", ExportLoadingTransforms.getFilesByPatterns())
-            .apply("Load from Datastore files", ExportLoadingTransforms.loadDataFromFiles());
+            .apply("Find Datastore files", Transforms.getFilesByPatterns())
+            .apply("Load from Datastore files", ExportLoadingTransforms.loadExportDataFromFiles());
 
     // Transform bytes to pojo for analysis
     PCollection<Entity> entities =
         taggedRecords.apply(
             "Raw records to Entity",
             ParDo.of(
-                new DoFn<KV<String, byte[]>, Entity>() {
+                new DoFn<VersionedEntity, Entity>() {
                   @ProcessElement
                   public void processElement(
-                      @Element KV<String, byte[]> kv, OutputReceiver<Entity> out) {
-                    out.output(parseBytes(kv.getValue()));
+                      @Element VersionedEntity versionedEntity, OutputReceiver<Entity> out) {
+                    out.output(versionedEntity.getEntity().get());
                   }
                 }));
 
@@ -192,10 +198,4 @@ public class ExportloadingTransformsTest implements Serializable {
 
     pipeline.run();
   }
-
-  private static Entity parseBytes(byte[] record) {
-    EntityProto proto = new EntityProto();
-    proto.parseFrom(record);
-    return EntityTranslator.createFromPb(proto);
-  }
 }

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -195,14 +195,10 @@ public class Spec11PipelineTest {
             new JSONObject()
                 .put("fullyQualifiedDomainName", "111.com")
                 .put("threatType", "MALWARE")
-                .put("threatEntryMetadata", "NONE")
-                .put("platformType", "WINDOWS")
                 .toString(),
             new JSONObject()
                 .put("fullyQualifiedDomainName", "222.com")
                 .put("threatType", "MALWARE")
-                .put("threatEntryMetadata", "NONE")
-                .put("platformType", "WINDOWS")
                 .toString());
   }
 

core/src/test/java/google/registry/flows/EppLifecycleDomainTest.java

@@ -127,9 +127,7 @@ public class EppLifecycleDomainTest extends EppTestCase {
             ImmutableMap.of(
                 "DOMAIN", "example.tld",
                 "CRDATE", "2000-06-01T00:02:00Z",
-                // TODO(mcilwain): The exp. date should be restored back to 2002-06-01T00:02:00Z,
-                // but this is old behavior of being 1 year after the moment of the restore.
-                "EXDATE", "2001-07-01T00:03:00Z",
+                "EXDATE", "2002-06-01T00:02:00Z",
                 "UPDATE", "2000-07-01T00:03:00Z"));
 
     assertThatLogoutSucceeds();
@@ -203,11 +201,7 @@ public class EppLifecycleDomainTest extends EppTestCase {
             ImmutableMap.of(
                 "DOMAIN", "example.tld",
                 "CRDATE", "2000-06-01T00:02:00Z",
-                // TODO(mcilwain): The exp. date should be 2003-06-01T00:02:00Z, the same as its
-                // value prior to the deletion, because the year that was taken off when the
-                // autorenew was canceled will be re-added in renewal during the restore.
-                // For now though, the current behavior is 1 year after restore.
-                "EXDATE", "2003-07-05T00:03:00Z",
+                "EXDATE", "2003-06-01T00:02:00Z",
                 "UPDATE", "2002-07-05T00:03:00Z"));
 
     assertThatLogoutSucceeds();
@@ -289,10 +283,7 @@ public class EppLifecycleDomainTest extends EppTestCase {
             ImmutableMap.of(
                 "DOMAIN", "example.tld",
                 "CRDATE", "2000-06-01T00:02:00Z",
-                // TODO(mcilwain): The exp. date should be 2002-06-01T00:02:00Z, which is the
-                // current registration expiration time on the (deleted) domain, but for now is
-                // 1 year after restore.
-                "EXDATE", "2001-06-20T00:00:00Z",
+                "EXDATE", "2002-06-01T00:02:00Z",
                 "UPDATE", "2000-06-20T00:00:00Z"));
 
     assertThatLogoutSucceeds();

core/src/test/java/google/registry/flows/EppLifecycleHostTest.java

@@ -24,7 +24,6 @@ import static google.registry.testing.EppMetricSubject.assertThat;
 import static google.registry.testing.HostResourceSubject.assertAboutHosts;
 
 import com.google.common.collect.ImmutableMap;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.testing.AppEngineRule;
@@ -221,7 +220,7 @@ public class EppLifecycleHostTest extends EppTestCase {
         loadByForeignKey(DomainBase.class, "example.bar.foo.tld", timeAfterCreates).get();
     assertAboutHosts()
         .that(exampleBarFooTldHost)
-        .hasSuperordinateDomain(Key.create(exampleBarFooTldDomain));
+        .hasSuperordinateDomain(exampleBarFooTldDomain.createVKey());
     assertThat(exampleBarFooTldDomain.getSubordinateHosts())
         .containsExactly("ns1.example.bar.foo.tld");
 
@@ -231,14 +230,14 @@ public class EppLifecycleHostTest extends EppTestCase {
         loadByForeignKey(DomainBase.class, "example.foo.tld", timeAfterCreates).get();
     assertAboutHosts()
         .that(exampleFooTldHost)
-        .hasSuperordinateDomain(Key.create(exampleFooTldDomain));
+        .hasSuperordinateDomain(exampleFooTldDomain.createVKey());
     assertThat(exampleFooTldDomain.getSubordinateHosts()).containsExactly("ns1.example.foo.tld");
 
     HostResource exampleTldHost =
         loadByForeignKey(HostResource.class, "ns1.example.tld", timeAfterCreates).get();
     DomainBase exampleTldDomain =
         loadByForeignKey(DomainBase.class, "example.tld", timeAfterCreates).get();
-    assertAboutHosts().that(exampleTldHost).hasSuperordinateDomain(Key.create(exampleTldDomain));
+    assertAboutHosts().that(exampleTldHost).hasSuperordinateDomain(exampleTldDomain.createVKey());
     assertThat(exampleTldDomain.getSubordinateHosts()).containsExactly("ns1.example.tld");
 
     assertThatLogoutSucceeds();

core/src/test/java/google/registry/flows/contact/ContactTransferRequestFlowTest.java

@@ -52,6 +52,7 @@ import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import org.joda.time.DateTime;
 import org.junit.Before;
 import org.junit.Test;
@@ -129,9 +130,13 @@ public class ContactTransferRequestFlowTest
     // poll messages, the approval notice ones for gaining and losing registrars.
     assertPollMessagesEqual(
         Iterables.filter(
-            ofy().load()
+            ofy()
+                .load()
                 // Use toArray() to coerce the type to something keys() will accept.
-                .keys(contact.getTransferData().getServerApproveEntities().toArray(new Key<?>[]{}))
+                .keys(
+                    contact.getTransferData().getServerApproveEntities().stream()
+                        .map(VKey::getOfyKey)
+                        .toArray(Key[]::new))
                 .values(),
             PollMessage.class),
         ImmutableList.of(gainingApproveMessage, losingApproveMessage));

core/src/test/java/google/registry/flows/custom/TestDomainPricingCustomLogic.java

@@ -40,7 +40,7 @@ public class TestDomainPricingCustomLogic extends DomainPricingCustomLogic {
   public FeesAndCredits customizeRenewPrice(RenewPriceParameters priceParameters) {
     return priceParameters.domainName().toString().startsWith("costly-renew")
         ? addCustomFee(
-            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.RENEW))
+            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.RENEW, true))
         : priceParameters.feesAndCredits();
   }
 
@@ -48,7 +48,7 @@ public class TestDomainPricingCustomLogic extends DomainPricingCustomLogic {
   public FeesAndCredits customizeTransferPrice(TransferPriceParameters priceParameters) {
     return priceParameters.domainName().toString().startsWith("expensive")
         ? addCustomFee(
-            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.TRANSFER))
+            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.TRANSFER, true))
         : priceParameters.feesAndCredits();
   }
 
@@ -56,7 +56,7 @@ public class TestDomainPricingCustomLogic extends DomainPricingCustomLogic {
   public FeesAndCredits customizeUpdatePrice(UpdatePriceParameters priceParameters) {
     return priceParameters.domainName().toString().startsWith("non-free-update")
         ? addCustomFee(
-            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.UPDATE))
+            priceParameters.feesAndCredits(), Fee.create(ONE_HUNDRED_BUCKS, FeeType.UPDATE, true))
         : priceParameters.feesAndCredits();
   }
 

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -93,6 +93,7 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.util.Map;
 import org.joda.money.Money;
@@ -668,13 +669,24 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
                 .setPendingTransferExpirationTime(clock.nowUtc())
                 .build());
     // The server-approve entities should all be deleted.
-    assertThat(ofy().load().key(oldTransferData.getServerApproveBillingEvent()).now()).isNull();
-    assertThat(ofy().load().key(oldTransferData.getServerApproveAutorenewEvent()).now()).isNull();
-    assertThat(ofy().load().key(oldTransferData.getServerApproveAutorenewPollMessage()).now())
+    assertThat(ofy().load().key(oldTransferData.getServerApproveBillingEvent().getOfyKey()).now())
+        .isNull();
+    assertThat(ofy().load().key(oldTransferData.getServerApproveAutorenewEvent().getOfyKey()).now())
+        .isNull();
+    assertThat(
+            ofy()
+                .load()
+                .key(oldTransferData.getServerApproveAutorenewPollMessage().getOfyKey())
+                .now())
         .isNull();
     assertThat(oldTransferData.getServerApproveEntities()).isNotEmpty(); // Just a sanity check.
     assertThat(
-            ofy().load().keys(oldTransferData.getServerApproveEntities().toArray(new Key<?>[] {})))
+            ofy()
+                .load()
+                .keys(
+                    oldTransferData.getServerApproveEntities().stream()
+                        .map(VKey::getOfyKey)
+                        .toArray(Key[]::new)))
         .isEmpty();
   }
 
@@ -719,7 +731,7 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
     persistResource(
         newHostResource("ns1." + getUniqueIdFromCommand())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(reloadResourceByForeignKey()))
+            .setSuperordinateDomain(reloadResourceByForeignKey().createVKey())
             .setDeletionTime(clock.nowUtc().minusDays(1))
             .build());
     clock.advanceOneMilli();
@@ -767,7 +779,7 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
         persistResource(
             newHostResource("ns1." + getUniqueIdFromCommand())
                 .asBuilder()
-                .setSuperordinateDomain(Key.create(reloadResourceByForeignKey()))
+                .setSuperordinateDomain(reloadResourceByForeignKey().createVKey())
                 .build());
     persistResource(
         domain.asBuilder().addSubordinateHost(subordinateHost.getFullyQualifiedHostName()).build());

core/src/test/java/google/registry/flows/domain/DomainInfoFlowTest.java

@@ -123,14 +123,14 @@ public class DomainInfoFlowTest extends ResourceFlowTestCase<DomainInfoFlow, Dom
                 .build());
     // Set the superordinate domain of ns1.example.com to example.com. In reality, this would have
     // happened in the flow that created it, but here we just overwrite it in Datastore.
-    host1 = persistResource(host1.asBuilder().setSuperordinateDomain(Key.create(domain)).build());
+    host1 = persistResource(host1.asBuilder().setSuperordinateDomain(domain.createVKey()).build());
     // Create a subordinate host that is not delegated to by anyone.
     host3 =
         persistResource(
             new HostResource.Builder()
                 .setFullyQualifiedHostName("ns2.example.tld")
                 .setRepoId("3FF-TLD")
-                .setSuperordinateDomain(Key.create(domain))
+                .setSuperordinateDomain(domain.createVKey())
                 .build());
     // Add the subordinate host keys to the existing domain.
     domain =

core/src/test/java/google/registry/flows/domain/DomainRestoreRequestFlowTest.java

@@ -72,6 +72,7 @@ import google.registry.model.reporting.DomainTransactionRecord.TransactionReport
 import google.registry.model.reporting.HistoryEntry;
 import java.util.Map;
 import org.joda.money.Money;
+import org.joda.time.DateTime;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -93,6 +94,10 @@ public class DomainRestoreRequestFlowTest
   }
 
   void persistPendingDeleteDomain() throws Exception {
+    persistPendingDeleteDomain(clock.nowUtc().plusYears(5).plusDays(45));
+  }
+
+  void persistPendingDeleteDomain(DateTime expirationTime) throws Exception {
     DomainBase domain = newDomainBase(getUniqueIdFromCommand());
     HistoryEntry historyEntry =
         persistResource(
@@ -103,7 +108,7 @@ public class DomainRestoreRequestFlowTest
     persistResource(
         domain
             .asBuilder()
-            .setRegistrationExpirationTime(clock.nowUtc().plusYears(5).plusDays(45))
+            .setRegistrationExpirationTime(expirationTime)
             .setDeletionTime(clock.nowUtc().plusDays(35))
             .addGracePeriod(
                 GracePeriod.create(
@@ -129,9 +134,10 @@ public class DomainRestoreRequestFlowTest
   }
 
   @Test
-  public void testSuccess() throws Exception {
+  public void testSuccess_expiryStillInFuture_notExtended() throws Exception {
     setEppInput("domain_update_restore_request.xml", ImmutableMap.of("DOMAIN", "example.tld"));
-    persistPendingDeleteDomain();
+    DateTime expirationTime = clock.nowUtc().plusYears(5).plusDays(45);
+    persistPendingDeleteDomain(expirationTime);
     assertTransactionalFlow(true);
     // Double check that we see a poll message in the future for when the delete happens.
     assertThat(getPollMessages("TheRegistrar", clock.nowUtc().plusMonths(1))).hasSize(1);
@@ -140,11 +146,79 @@ public class DomainRestoreRequestFlowTest
     HistoryEntry historyEntryDomainRestore =
         getOnlyHistoryEntryOfType(domain, HistoryEntry.Type.DOMAIN_RESTORE);
     assertThat(ofy().load().key(domain.getAutorenewBillingEvent()).now().getEventTime())
-        .isEqualTo(clock.nowUtc().plusYears(1));
+        .isEqualTo(expirationTime);
+    assertAboutDomains()
+        .that(domain)
+        // New expiration time should be the same as from before the deletion.
+        .hasRegistrationExpirationTime(expirationTime)
+        .and()
+        .doesNotHaveStatusValue(StatusValue.PENDING_DELETE)
+        .and()
+        .hasDeletionTime(END_OF_TIME)
+        .and()
+        .hasOneHistoryEntryEachOfTypes(
+            HistoryEntry.Type.DOMAIN_DELETE, HistoryEntry.Type.DOMAIN_RESTORE)
+        .and()
+        .hasLastEppUpdateTime(clock.nowUtc())
+        .and()
+        .hasLastEppUpdateClientId("TheRegistrar");
+    assertThat(domain.getGracePeriods()).isEmpty();
+    assertDnsTasksEnqueued("example.tld");
+    // The poll message for the delete should now be gone. The only poll message should be the new
+    // autorenew poll message.
+    assertPollMessages(
+        "TheRegistrar",
+        new PollMessage.Autorenew.Builder()
+            .setTargetId("example.tld")
+            .setClientId("TheRegistrar")
+            .setEventTime(domain.getRegistrationExpirationTime())
+            .setAutorenewEndTime(END_OF_TIME)
+            .setMsg("Domain was auto-renewed.")
+            .setParent(historyEntryDomainRestore)
+            .build());
+    // There should be a onetime for the restore and a new recurring billing event, but no renew
+    // onetime.
+    assertBillingEvents(
+        new BillingEvent.Recurring.Builder()
+            .setReason(Reason.RENEW)
+            .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
+            .setTargetId("example.tld")
+            .setClientId("TheRegistrar")
+            .setEventTime(expirationTime)
+            .setRecurrenceEndTime(END_OF_TIME)
+            .setParent(historyEntryDomainRestore)
+            .build(),
+        new BillingEvent.OneTime.Builder()
+            .setReason(Reason.RESTORE)
+            .setTargetId("example.tld")
+            .setClientId("TheRegistrar")
+            .setCost(Money.of(USD, 17))
+            .setPeriodYears(1)
+            .setEventTime(clock.nowUtc())
+            .setBillingTime(clock.nowUtc())
+            .setParent(historyEntryDomainRestore)
+            .build());
+  }
+
+  @Test
+  public void testSuccess_expiryInPast_extendedByOneYear() throws Exception {
+    setEppInput("domain_update_restore_request.xml", ImmutableMap.of("DOMAIN", "example.tld"));
+    DateTime expirationTime = clock.nowUtc().minusDays(20);
+    DateTime newExpirationTime = expirationTime.plusYears(1);
+    persistPendingDeleteDomain(expirationTime);
+    assertTransactionalFlow(true);
+    // Double check that we see a poll message in the future for when the delete happens.
+    assertThat(getPollMessages("TheRegistrar", clock.nowUtc().plusMonths(1))).hasSize(1);
+    runFlowAssertResponse(loadFile("generic_success_response.xml"));
+    DomainBase domain = reloadResourceByForeignKey();
+    HistoryEntry historyEntryDomainRestore =
+        getOnlyHistoryEntryOfType(domain, HistoryEntry.Type.DOMAIN_RESTORE);
+    assertThat(ofy().load().key(domain.getAutorenewBillingEvent()).now().getEventTime())
+        .isEqualTo(newExpirationTime);
     assertAboutDomains()
         .that(domain)
         // New expiration time should be exactly a year from now.
-        .hasRegistrationExpirationTime(clock.nowUtc().plusYears(1))
+        .hasRegistrationExpirationTime(newExpirationTime)
         .and()
         .doesNotHaveStatusValue(StatusValue.PENDING_DELETE)
         .and()
@@ -178,7 +252,7 @@ public class DomainRestoreRequestFlowTest
             .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
             .setTargetId("example.tld")
             .setClientId("TheRegistrar")
-            .setEventTime(domain.getRegistrationExpirationTime())
+            .setEventTime(newExpirationTime)
             .setRecurrenceEndTime(END_OF_TIME)
             .setParent(historyEntryDomainRestore)
             .build(),

core/src/test/java/google/registry/flows/domain/DomainTransferFlowTestCase.java

@@ -29,7 +29,6 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.flows.Flow;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.model.EppResource;
@@ -109,15 +108,16 @@ public class DomainTransferFlowTestCase<F extends Flow, R extends EppResource>
             clock.nowUtc(),
             DateTime.parse("1999-04-03T22:00:00.0Z"),
             REGISTRATION_EXPIRATION_TIME);
-    subordinateHost = persistResource(
-        new HostResource.Builder()
-            .setRepoId("2-".concat(Ascii.toUpperCase(tld)))
-            .setFullyQualifiedHostName("ns1." + label + "." + tld)
-            .setPersistedCurrentSponsorClientId("TheRegistrar")
-            .setCreationClientId("TheRegistrar")
-            .setCreationTimeForTest(DateTime.parse("1999-04-03T22:00:00.0Z"))
-            .setSuperordinateDomain(Key.create(domain))
-            .build());
+    subordinateHost =
+        persistResource(
+            new HostResource.Builder()
+                .setRepoId("2-".concat(Ascii.toUpperCase(tld)))
+                .setFullyQualifiedHostName("ns1." + label + "." + tld)
+                .setPersistedCurrentSponsorClientId("TheRegistrar")
+                .setCreationClientId("TheRegistrar")
+                .setCreationTimeForTest(DateTime.parse("1999-04-03T22:00:00.0Z"))
+                .setSuperordinateDomain(domain.createVKey())
+                .build());
     domain =
         persistResource(
             domain

core/src/test/java/google/registry/flows/domain/DomainTransferRequestFlowTest.java

@@ -27,6 +27,7 @@ import static google.registry.model.registry.Registry.TldState.QUIET_PERIOD;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.TRANSFER_SUCCESSFUL;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_CREATE;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_TRANSFER_REQUEST;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatastoreHelper.assertBillingEvents;
 import static google.registry.testing.DatastoreHelper.assertBillingEventsEqual;
 import static google.registry.testing.DatastoreHelper.assertPollMessagesEqual;
@@ -290,13 +291,13 @@ public class DomainTransferRequestFlowTest
     // Assert that the domain's TransferData server-approve billing events match the above.
     if (expectTransferBillingEvent) {
       assertBillingEventsEqual(
-          ofy().load().key(domain.getTransferData().getServerApproveBillingEvent()).now(),
+          tm().load(domain.getTransferData().getServerApproveBillingEvent()),
           optionalTransferBillingEvent.get());
     } else {
       assertThat(domain.getTransferData().getServerApproveBillingEvent()).isNull();
     }
     assertBillingEventsEqual(
-        ofy().load().key(domain.getTransferData().getServerApproveAutorenewEvent()).now(),
+        tm().load(domain.getTransferData().getServerApproveAutorenewEvent()),
         gainingClientAutorenew);
     // Assert that the full set of server-approve billing events is exactly the extra ones plus
     // the transfer billing event (if present) and the gaining client autorenew.
@@ -309,7 +310,10 @@ public class DomainTransferRequestFlowTest
             ofy()
                 .load()
                 // Use toArray() to coerce the type to something keys() will accept.
-                .keys(domain.getTransferData().getServerApproveEntities().toArray(new Key<?>[] {}))
+                .keys(
+                    domain.getTransferData().getServerApproveEntities().stream()
+                        .map(VKey::getOfyKey)
+                        .toArray(Key[]::new))
                 .values(),
             BillingEvent.class),
         Sets.union(expectedServeApproveBillingEvents, extraBillingEvents));
@@ -410,7 +414,7 @@ public class DomainTransferRequestFlowTest
 
     // Assert that the poll messages show up in the TransferData server approve entities.
     assertPollMessagesEqual(
-        ofy().load().key(domain.getTransferData().getServerApproveAutorenewPollMessage()).now(),
+        tm().load(domain.getTransferData().getServerApproveAutorenewPollMessage()),
         autorenewPollMessage);
     // Assert that the full set of server-approve poll messages is exactly the server approve
     // OneTime messages to gaining and losing registrars plus the gaining client autorenew.
@@ -419,7 +423,10 @@ public class DomainTransferRequestFlowTest
             ofy()
                 .load()
                 // Use toArray() to coerce the type to something keys() will accept.
-                .keys(domain.getTransferData().getServerApproveEntities().toArray(new Key<?>[] {}))
+                .keys(
+                    domain.getTransferData().getServerApproveEntities().stream()
+                        .map(VKey::getOfyKey)
+                        .toArray(Key[]::new))
                 .values(),
             PollMessage.class),
         ImmutableList.of(

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -389,8 +389,8 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
     HostResource existingHost =
         loadByForeignKey(HostResource.class, "ns1.example.tld", clock.nowUtc()).get();
     addedHost = loadByForeignKey(HostResource.class, "ns2.example.tld", clock.nowUtc()).get();
-    assertThat(existingHost.getSuperordinateDomain()).isEqualTo(Key.create(domain));
-    assertThat(addedHost.getSuperordinateDomain()).isEqualTo(Key.create(domain));
+    assertThat(existingHost.getSuperordinateDomain()).isEqualTo(domain.createVKey());
+    assertThat(addedHost.getSuperordinateDomain()).isEqualTo(domain.createVKey());
   }
 
   @Test

core/src/test/java/google/registry/flows/host/HostCreateFlowTest.java

@@ -34,7 +34,6 @@ import static org.junit.Assert.assertThrows;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
 import google.registry.flows.FlowUtils.IpAddressVersionMismatchException;
 import google.registry.flows.ResourceFlowTestCase;
@@ -119,7 +118,7 @@ public class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, Hos
     HostResource host = reloadResourceByForeignKey();
     DomainBase superordinateDomain =
         loadByForeignKey(DomainBase.class, "example.tld", clock.nowUtc()).get();
-    assertAboutHosts().that(host).hasSuperordinateDomain(Key.create(superordinateDomain));
+    assertAboutHosts().that(host).hasSuperordinateDomain(superordinateDomain.createVKey());
     assertThat(superordinateDomain.getSubordinateHosts()).containsExactly("ns1.example.tld");
     assertDnsTasksEnqueued("ns1.example.tld");
   }
@@ -148,7 +147,7 @@ public class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, Hos
     HostResource host = reloadResourceByForeignKey();
     DomainBase superordinateDomain =
         loadByForeignKey(DomainBase.class, "example.tld", clock.nowUtc()).get();
-    assertAboutHosts().that(host).hasSuperordinateDomain(Key.create(superordinateDomain));
+    assertAboutHosts().that(host).hasSuperordinateDomain(superordinateDomain.createVKey());
     assertThat(superordinateDomain.getSubordinateHosts()).containsExactly("ns1.example.tld");
     assertDnsTasksEnqueued("ns1.example.tld");
   }

core/src/test/java/google/registry/flows/host/HostDeleteFlowTest.java

@@ -29,7 +29,6 @@ import static org.junit.Assert.assertThrows;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
@@ -186,7 +185,7 @@ public class HostDeleteFlowTest extends ResourceFlowTestCase<HostDeleteFlow, Hos
         newHostResource("ns1.example.tld")
             .asBuilder()
             .setPersistedCurrentSponsorClientId("NewRegistrar") // Shouldn't hurt.
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .build());
     clock.advanceOneMilli();
     runFlowAssertResponse(loadFile("host_delete_response.xml"));
@@ -206,7 +205,7 @@ public class HostDeleteFlowTest extends ResourceFlowTestCase<HostDeleteFlow, Hos
         newHostResource("ns1.example.tld")
             .asBuilder()
             .setPersistedCurrentSponsorClientId("TheRegistrar") // Shouldn't help.
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .build());
     EppException thrown = assertThrows(ResourceNotOwnedException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
@@ -239,7 +238,7 @@ public class HostDeleteFlowTest extends ResourceFlowTestCase<HostDeleteFlow, Hos
         newHostResource("ns1.example.tld")
             .asBuilder()
             .setPersistedCurrentSponsorClientId("TheRegistrar") // Shouldn't hurt.
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .build());
     clock.advanceOneMilli();
     runFlowAssertResponse(loadFile("host_delete_response.xml"));
@@ -272,7 +271,7 @@ public class HostDeleteFlowTest extends ResourceFlowTestCase<HostDeleteFlow, Hos
         newHostResource("ns1.example.tld")
             .asBuilder()
             .setPersistedCurrentSponsorClientId("NewRegistrar") // Shouldn't help.
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .build());
     EppException thrown = assertThrows(ResourceNotOwnedException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();

core/src/test/java/google/registry/flows/host/HostInfoFlowTest.java

@@ -25,7 +25,6 @@ import static org.junit.Assert.assertThrows;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.net.InetAddresses;
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
@@ -118,7 +117,7 @@ public class HostInfoFlowTest extends ResourceFlowTestCase<HostInfoFlow, HostRes
         persistHostResource()
             .asBuilder()
             .setRepoId("CEEF-FOOBAR")
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .setLastSuperordinateChange(lastSuperordinateChange)
             .build());
     assertTransactionalFlow(false);

core/src/test/java/google/registry/flows/host/HostUpdateFlowTest.java

@@ -220,7 +220,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         .that(reloadResourceByForeignKey())
         .hasLastSuperordinateChange(oldHost.getLastSuperordinateChange())
         .and()
-        .hasSuperordinateDomain(Key.create(domain))
+        .hasSuperordinateDomain(domain.createVKey())
         .and()
         .hasPersistedCurrentSponsorClientId("TheRegistrar")
         .and()
@@ -246,7 +246,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         .that(reloadResourceByForeignKey())
         .hasLastSuperordinateChange(oldHost.getLastSuperordinateChange())
         .and()
-        .hasSuperordinateDomain(Key.create(domain))
+        .hasSuperordinateDomain(domain.createVKey())
         .and()
         .hasPersistedCurrentSponsorClientId("NewRegistrar")
         .and()
@@ -279,7 +279,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     HostResource renamedHost = doSuccessfulTest();
     assertAboutHosts()
         .that(renamedHost)
-        .hasSuperordinateDomain(Key.create(domain))
+        .hasSuperordinateDomain(domain.createVKey())
         .and()
         .hasLastSuperordinateChange(oldHost.getLastSuperordinateChange())
         .and()
@@ -313,7 +313,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     DateTime now = clock.nowUtc();
     assertAboutHosts()
         .that(renamedHost)
-        .hasSuperordinateDomain(Key.create(example))
+        .hasSuperordinateDomain(example.createVKey())
         .and()
         .hasLastSuperordinateChange(now)
         .and()
@@ -350,7 +350,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     DateTime now = clock.nowUtc();
     assertAboutHosts()
         .that(renamedHost)
-        .hasSuperordinateDomain(Key.create(tldDomain))
+        .hasSuperordinateDomain(tldDomain.createVKey())
         .and()
         .hasLastSuperordinateChange(now)
         .and()
@@ -432,7 +432,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     DateTime now = clock.nowUtc();
     assertAboutHosts()
         .that(renamedHost)
-        .hasSuperordinateDomain(Key.create(domain))
+        .hasSuperordinateDomain(domain.createVKey())
         .and()
         .hasLastSuperordinateChange(now)
         .and()
@@ -514,7 +514,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     persistResource(
         newHostResource(oldHostName())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(foo))
+            .setSuperordinateDomain(foo.createVKey())
             .setLastTransferTime(null)
             .build());
     persistResource(foo.asBuilder().setSubordinateHosts(ImmutableSet.of(oldHostName())).build());
@@ -551,7 +551,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         persistResource(
             newHostResource(oldHostName())
                 .asBuilder()
-                .setSuperordinateDomain(Key.create(domain))
+                .setSuperordinateDomain(domain.createVKey())
                 .setLastTransferTime(clock.nowUtc().minusDays(20))
                 .setLastSuperordinateChange(clock.nowUtc().minusDays(3))
                 .build());
@@ -586,7 +586,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     persistResource(
         newHostResource(oldHostName())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(foo))
+            .setSuperordinateDomain(foo.createVKey())
             .setLastTransferTime(lastTransferTime)
             .setLastSuperordinateChange(clock.nowUtc().minusDays(3))
             .build());
@@ -617,7 +617,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     persistResource(
         newHostResource(oldHostName())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(foo))
+            .setSuperordinateDomain(foo.createVKey())
             .setLastTransferTime(lastTransferTime)
             .setLastSuperordinateChange(clock.nowUtc().minusDays(10))
             .build());
@@ -649,7 +649,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     persistResource(
         newHostResource(oldHostName())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(foo))
+            .setSuperordinateDomain(foo.createVKey())
             .setLastTransferTime(null)
             .setLastSuperordinateChange(clock.nowUtc().minusDays(3))
             .build());
@@ -675,7 +675,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     persistResource(
         newHostResource(oldHostName())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .build());
     DateTime lastTransferTime = clock.nowUtc().minusDays(2);
     persistResource(
@@ -712,7 +712,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     persistResource(
         newHostResource(oldHostName())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .setLastTransferTime(lastTransferTime)
             .setLastSuperordinateChange(clock.nowUtc().minusDays(4))
             .build());
@@ -747,7 +747,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     persistResource(
         newHostResource(oldHostName())
             .asBuilder()
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .setLastTransferTime(clock.nowUtc().minusDays(12))
             .setLastSuperordinateChange(clock.nowUtc().minusDays(4))
             .build());
@@ -1017,7 +1017,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         newHostResource(oldHostName())
             .asBuilder()
             .setStatusValues(ImmutableSet.of(StatusValue.CLIENT_UPDATE_PROHIBITED))
-            .setSuperordinateDomain(Key.create(persistActiveDomain("example.tld")))
+            .setSuperordinateDomain(persistActiveDomain("example.tld").createVKey())
             .build());
     EppException thrown =
         assertThrows(ResourceHasClientUpdateProhibitedException.class, this::runFlow);
@@ -1031,7 +1031,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         newHostResource(oldHostName())
             .asBuilder()
             .setStatusValues(ImmutableSet.of(StatusValue.SERVER_UPDATE_PROHIBITED))
-            .setSuperordinateDomain(Key.create(persistActiveDomain("example.tld")))
+            .setSuperordinateDomain(persistActiveDomain("example.tld").createVKey())
             .build());
     ResourceStatusProhibitsOperationException thrown =
         assertThrows(ResourceStatusProhibitsOperationException.class, this::runFlow);
@@ -1045,7 +1045,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         newHostResource(oldHostName())
             .asBuilder()
             .setStatusValues(ImmutableSet.of(StatusValue.PENDING_DELETE))
-            .setSuperordinateDomain(Key.create(persistActiveDomain("example.tld")))
+            .setSuperordinateDomain(persistActiveDomain("example.tld").createVKey())
             .build());
     ResourceStatusProhibitsOperationException thrown =
         assertThrows(ResourceStatusProhibitsOperationException.class, this::runFlow);
@@ -1105,7 +1105,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         newHostResource("ns1.example.tld")
             .asBuilder()
             .setPersistedCurrentSponsorClientId("TheRegistrar") // Shouldn't hurt.
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
             .build());
 
@@ -1127,7 +1127,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         newHostResource("ns1.example.tld")
             .asBuilder()
             .setPersistedCurrentSponsorClientId("NewRegistrar") // Shouldn't help.
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
             .build());
 
@@ -1145,7 +1145,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         newHostResource("ns1.example.tld")
             .asBuilder()
             .setPersistedCurrentSponsorClientId("TheRegistrar") // Shouldn't hurt.
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
             .build());
 
@@ -1163,7 +1163,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         newHostResource("ns1.example.tld")
             .asBuilder()
             .setPersistedCurrentSponsorClientId("TheRegistrar") // Shouldn't help.
-            .setSuperordinateDomain(Key.create(domain))
+            .setSuperordinateDomain(domain.createVKey())
             .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
             .build());
 
@@ -1228,7 +1228,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     persistResource(
         newHostResource("ns1.example.foo")
             .asBuilder()
-            .setSuperordinateDomain(Key.create(superordinate))
+            .setSuperordinateDomain(superordinate.createVKey())
             .setPersistedCurrentSponsorClientId("NewRegistrar")
             .build());
 

core/src/test/java/google/registry/model/contact/ContactResourceTest.java

@@ -29,7 +29,6 @@ import static org.junit.Assert.assertThrows;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.EntityTestCase;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.contact.Disclose.PostalInfoChoice;
@@ -114,7 +113,7 @@ public class ContactResourceTest extends EntityTestCase {
                     .setLosingClientId("losing")
                     .setPendingTransferExpirationTime(fakeClock.nowUtc())
                     .setServerApproveEntities(
-                        ImmutableSet.of(Key.create(BillingEvent.OneTime.class, 1)))
+                        ImmutableSet.of(VKey.createOfy(BillingEvent.OneTime.class, 1)))
                     .setTransferRequestTime(fakeClock.nowUtc())
                     .setTransferStatus(TransferStatus.SERVER_APPROVED)
                     .setTransferRequestTrid(Trid.create("client-trid", "server-trid"))
@@ -134,6 +133,8 @@ public class ContactResourceTest extends EntityTestCase {
     saveRegistrar("registrar1");
     saveRegistrar("registrar2");
     saveRegistrar("registrar3");
+    saveRegistrar("gaining");
+    saveRegistrar("losing");
     jpaTm().transact(() -> jpaTm().saveNew(originalContact));
     ContactResource persisted =
         jpaTm()
@@ -141,15 +142,17 @@ public class ContactResourceTest extends EntityTestCase {
                 () ->
                     jpaTm()
                         .load(VKey.createSql(ContactResource.class, originalContact.getRepoId())));
-    // TODO(b/153378849): Remove the hard code for postal info after resolving the issue that
-    // @PostLoad doesn't work in Address
+
     ContactResource fixed =
         originalContact
             .asBuilder()
             .setCreationTime(persisted.getCreationTime())
-            .setInternationalizedPostalInfo(persisted.getInternationalizedPostalInfo())
-            .setLocalizedPostalInfo(persisted.getLocalizedPostalInfo())
-            .setTransferData(null)
+            .setTransferData(
+                originalContact
+                    .getTransferData()
+                    .asBuilder()
+                    .setServerApproveEntities(null)
+                    .build())
             .build();
     assertThat(persisted).isEqualTo(fixed);
   }

core/src/test/java/google/registry/model/domain/DomainBaseTest.java

@@ -76,7 +76,7 @@ public class DomainBaseTest extends EntityTestCase {
         persistResource(
                 new HostResource.Builder()
                     .setFullyQualifiedHostName("ns1.example.com")
-                    .setSuperordinateDomain(domainKey)
+                    .setSuperordinateDomain(VKey.createOfy(DomainBase.class, domainKey))
                     .setRepoId("1-COM")
                     .build())
             .createVKey();
@@ -138,10 +138,16 @@ public class DomainBaseTest extends EntityTestCase {
                             .setLosingClientId("losing")
                             .setPendingTransferExpirationTime(fakeClock.nowUtc())
                             .setServerApproveEntities(
-                                ImmutableSet.of(oneTimeBillKey, recurringBillKey, autorenewPollKey))
-                            .setServerApproveBillingEvent(oneTimeBillKey)
-                            .setServerApproveAutorenewEvent(recurringBillKey)
-                            .setServerApproveAutorenewPollMessage(autorenewPollKey)
+                                ImmutableSet.of(
+                                    VKey.createOfy(BillingEvent.OneTime.class, oneTimeBillKey),
+                                    VKey.createOfy(BillingEvent.Recurring.class, recurringBillKey),
+                                    VKey.createOfy(PollMessage.Autorenew.class, autorenewPollKey)))
+                            .setServerApproveBillingEvent(
+                                VKey.createOfy(BillingEvent.OneTime.class, oneTimeBillKey))
+                            .setServerApproveAutorenewEvent(
+                                VKey.createOfy(BillingEvent.Recurring.class, recurringBillKey))
+                            .setServerApproveAutorenewPollMessage(
+                                VKey.createOfy(PollMessage.Autorenew.class, autorenewPollKey))
                             .setTransferRequestTime(fakeClock.nowUtc().plusDays(1))
                             .setTransferStatus(TransferStatus.SERVER_APPROVED)
                             .setTransferRequestTrid(Trid.create("client-trid", "server-trid"))
@@ -363,8 +369,8 @@ public class DomainBaseTest extends EntityTestCase {
                     .setTransferRequestTime(fakeClock.nowUtc().minusDays(4))
                     .setPendingTransferExpirationTime(fakeClock.nowUtc().plusDays(1))
                     .setGainingClientId("winner")
-                    .setServerApproveBillingEvent(Key.create(transferBillingEvent))
-                    .setServerApproveEntities(ImmutableSet.of(Key.create(transferBillingEvent)))
+                    .setServerApproveBillingEvent(transferBillingEvent.createVKey())
+                    .setServerApproveEntities(ImmutableSet.of(transferBillingEvent.createVKey()))
                     .build())
             .addGracePeriod(
                 // Okay for billing event to be null since the point of this grace period is just
@@ -375,7 +381,7 @@ public class DomainBaseTest extends EntityTestCase {
     DomainBase afterTransfer = domain.cloneProjectedAtTime(fakeClock.nowUtc().plusDays(1));
     DateTime newExpirationTime = oldExpirationTime.plusYears(1);
     Key<BillingEvent.Recurring> serverApproveAutorenewEvent =
-        domain.getTransferData().getServerApproveAutorenewEvent();
+        domain.getTransferData().getServerApproveAutorenewEvent().getOfyKey();
     assertTransferred(afterTransfer, newExpirationTime, serverApproveAutorenewEvent);
     assertThat(afterTransfer.getGracePeriods())
         .containsExactly(
@@ -746,8 +752,10 @@ public class DomainBaseTest extends EntityTestCase {
             .setPendingTransferExpirationTime(transferExpirationTime)
             .setTransferStatus(TransferStatus.PENDING)
             .setGainingClientId("TheRegistrar")
-            .setServerApproveAutorenewEvent(recurringBillKey)
-            .setServerApproveBillingEvent(oneTimeBillKey)
+            .setServerApproveAutorenewEvent(
+                VKey.createOfy(BillingEvent.Recurring.class, recurringBillKey))
+            .setServerApproveBillingEvent(
+                VKey.createOfy(BillingEvent.OneTime.class, oneTimeBillKey))
             .build();
     domain =
         persistResource(

core/src/test/java/google/registry/model/host/HostResourceTest.java

@@ -26,7 +26,6 @@ import static org.junit.Assert.assertThrows;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.net.InetAddresses;
-import com.googlecode.objectify.Key;
 import google.registry.model.EntityTestCase;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.DomainBase;
@@ -34,6 +33,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.persistence.VKey;
 import org.joda.time.DateTime;
 import org.junit.Before;
 import org.junit.Test;
@@ -63,7 +63,7 @@ public class HostResourceTest extends EntityTestCase {
                         .setLosingClientId("losing")
                         .setPendingTransferExpirationTime(fakeClock.nowUtc())
                         .setServerApproveEntities(
-                            ImmutableSet.of(Key.create(BillingEvent.OneTime.class, 1)))
+                            ImmutableSet.of(VKey.createOfy(BillingEvent.OneTime.class, 1)))
                         .setTransferRequestTime(fakeClock.nowUtc())
                         .setTransferStatus(TransferStatus.SERVER_APPROVED)
                         .setTransferRequestTrid(Trid.create("client-trid", "server-trid"))
@@ -81,7 +81,7 @@ public class HostResourceTest extends EntityTestCase {
                     .setLastTransferTime(fakeClock.nowUtc())
                     .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
                     .setStatusValues(ImmutableSet.of(StatusValue.OK))
-                    .setSuperordinateDomain(Key.create(domain))
+                    .setSuperordinateDomain(domain.createVKey())
                     .build()));
   }
 
@@ -228,7 +228,7 @@ public class HostResourceTest extends EntityTestCase {
                     .setLastEppUpdateClientId("another registrar")
                     .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
                     .setStatusValues(ImmutableSet.of(StatusValue.OK))
-                    .setSuperordinateDomain(Key.create(domain))
+                    .setSuperordinateDomain(domain.createVKey())
                     .build()));
     assertThat(host.computeLastTransferTime(domain)).isNull();
   }

core/src/test/java/google/registry/model/transfer/TransferDataTest.java

@@ -18,11 +18,11 @@ import static com.google.common.truth.Truth.assertThat;
 import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.Period;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.poll.PollMessage;
+import google.registry.persistence.VKey;
 import google.registry.testing.AppEngineRule;
 import org.joda.time.DateTime;
 import org.junit.Before;
@@ -40,19 +40,19 @@ public class TransferDataTest {
 
   private final DateTime now = DateTime.now(UTC);
 
-  private Key<BillingEvent.OneTime> transferBillingEventKey;
-  private Key<BillingEvent.Cancellation> otherServerApproveBillingEventKey;
-  private Key<BillingEvent.Recurring> recurringBillingEventKey;
-  private Key<PollMessage.Autorenew> autorenewPollMessageKey;
-  private Key<PollMessage.OneTime> otherServerApprovePollMessageKey;
+  private VKey<BillingEvent.OneTime> transferBillingEventKey;
+  private VKey<BillingEvent.Cancellation> otherServerApproveBillingEventKey;
+  private VKey<BillingEvent.Recurring> recurringBillingEventKey;
+  private VKey<PollMessage.Autorenew> autorenewPollMessageKey;
+  private VKey<PollMessage.OneTime> otherServerApprovePollMessageKey;
 
   @Before
   public void setUp() {
-    transferBillingEventKey = Key.create(BillingEvent.OneTime.class, 12345);
-    otherServerApproveBillingEventKey = Key.create(BillingEvent.Cancellation.class, 2468);
-    recurringBillingEventKey = Key.create(BillingEvent.Recurring.class, 13579);
-    autorenewPollMessageKey = Key.create(PollMessage.Autorenew.class, 67890);
-    otherServerApprovePollMessageKey = Key.create(PollMessage.OneTime.class, 314159);
+    transferBillingEventKey = VKey.createOfy(BillingEvent.OneTime.class, 12345);
+    otherServerApproveBillingEventKey = VKey.createOfy(BillingEvent.Cancellation.class, 2468);
+    recurringBillingEventKey = VKey.createOfy(BillingEvent.Recurring.class, 13579);
+    autorenewPollMessageKey = VKey.createOfy(PollMessage.Autorenew.class, 67890);
+    otherServerApprovePollMessageKey = VKey.createOfy(PollMessage.OneTime.class, 314159);
   }
 
   @Test
@@ -67,7 +67,8 @@ public class TransferDataTest {
             .setTransferPeriod(Period.create(5, Period.Unit.YEARS))
             .build();
     TransferData fullTransferData =
-        constantTransferData.asBuilder()
+        constantTransferData
+            .asBuilder()
             .setPendingTransferExpirationTime(now)
             .setTransferStatus(TransferStatus.PENDING)
             .setServerApproveEntities(

core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java

@@ -0,0 +1,86 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.net.InetAddresses;
+import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
+import google.registry.schema.replay.EntityTest.EntityForTesting;
+import google.registry.testing.AppEngineRule;
+import java.net.InetAddress;
+import java.util.Set;
+import javax.annotation.Nullable;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit tests for {@link google.registry.persistence.converter.InetAddressSetConverter}. */
+public class InetAddressSetConverterTest {
+
+  @RegisterExtension
+  public final AppEngineRule appEngine =
+      AppEngineRule.builder()
+          .withDatastoreAndCloudSql()
+          .withJpaUnitTestEntities(InetAddressSetTestEntity.class)
+          .build();
+
+  @Test
+  public void roundTripConversion_returnsSameAddresses() {
+    verifySaveAndLoad(
+        ImmutableSet.of(
+            InetAddresses.forString("0.0.0.0"),
+            InetAddresses.forString("192.168.0.1"),
+            InetAddresses.forString("2001:41d0:1:a41e:0:0:0:1"),
+            InetAddresses.forString("2041:0:140F::875B:131B")));
+  }
+
+  @Test
+  public void roundTrip_emptySet() {
+    verifySaveAndLoad(ImmutableSet.of());
+  }
+
+  @Test
+  public void roundTrip_null() {
+    verifySaveAndLoad(null);
+  }
+
+  private void verifySaveAndLoad(@Nullable Set<InetAddress> inetAddresses) {
+    InetAddressSetTestEntity testEntity = new InetAddressSetTestEntity(inetAddresses);
+    jpaTm().transact(() -> jpaTm().saveNew(testEntity));
+    InetAddressSetTestEntity persisted =
+        jpaTm().transact(() -> jpaTm().load(VKey.createSql(InetAddressSetTestEntity.class, "id")));
+    assertThat(persisted.addresses).isEqualTo(inetAddresses);
+  }
+
+  @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference.
+  @EntityForTesting
+  private static class InetAddressSetTestEntity extends ImmutableObject {
+
+    @Id String name = "id";
+
+    Set<InetAddress> addresses;
+
+    private InetAddressSetTestEntity() {}
+
+    private InetAddressSetTestEntity(Set<InetAddress> addresses) {
+      this.addresses = addresses;
+    }
+  }
+}

core/src/test/java/google/registry/persistence/transaction/JpaTestRules.java

@@ -71,7 +71,7 @@ public class JpaTestRules {
         Optional<String> initScriptPath,
         ImmutableList<Class> extraEntityClasses,
         ImmutableMap<String, String> userProperties) {
-      super(clock, initScriptPath, extraEntityClasses, userProperties);
+      super(clock, initScriptPath, false, extraEntityClasses, userProperties);
     }
 
     @Override

core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerRule.java

@@ -15,6 +15,7 @@
 package google.registry.persistence.transaction;
 
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertWithMessage;
 import static org.testcontainers.containers.PostgreSQLContainer.POSTGRESQL_PORT;
 
@@ -41,12 +42,14 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.persistence.Entity;
 import javax.persistence.EntityManagerFactory;
 import org.hibernate.cfg.Environment;
 import org.hibernate.jpa.boot.internal.ParsedPersistenceXmlDescriptor;
@@ -94,7 +97,25 @@ abstract class JpaTransactionManagerRule extends ExternalResource {
   // Hash of the ORM entity names requested by this rule instance.
   private int entityHash;
 
-  protected JpaTransactionManagerRule(
+  // Whether to create nomulus tables in the test db. Right now, only the JpaTestRules set this to
+  // false.
+  private boolean includeNomulusSchema = true;
+
+  JpaTransactionManagerRule(
+      Clock clock,
+      Optional<String> initScriptPath,
+      boolean includeNomulusSchema,
+      ImmutableList<Class> extraEntityClasses,
+      ImmutableMap<String, String> userProperties) {
+    this.clock = clock;
+    this.initScriptPath = initScriptPath;
+    this.includeNomulusSchema = includeNomulusSchema;
+    this.extraEntityClasses = extraEntityClasses;
+    this.userProperties = userProperties;
+    this.entityHash = getOrmEntityHash(initScriptPath, extraEntityClasses);
+  }
+
+  JpaTransactionManagerRule(
       Clock clock,
       Optional<String> initScriptPath,
       ImmutableList<Class> extraEntityClasses,
@@ -277,7 +298,7 @@ abstract class JpaTransactionManagerRule extends ExternalResource {
   }
 
   /** Constructs the {@link EntityManagerFactory} instance. */
-  private static EntityManagerFactory createEntityManagerFactory(
+  private EntityManagerFactory createEntityManagerFactory(
       String jdbcUrl,
       String username,
       String password,
@@ -293,6 +314,24 @@ abstract class JpaTransactionManagerRule extends ExternalResource {
     ParsedPersistenceXmlDescriptor descriptor =
         PersistenceXmlUtility.getParsedPersistenceXmlDescriptor();
 
+    // If we don't include the nomulus schema, remove all entity classes in the descriptor but keep
+    // other settings like the converter classes.
+    if (!includeNomulusSchema) {
+      List<String> nonEntityClasses =
+          descriptor.getManagedClassNames().stream()
+              .filter(
+                  classString -> {
+                    try {
+                      return !Class.forName(classString).isAnnotationPresent(Entity.class);
+                    } catch (ClassNotFoundException e) {
+                      throw new IllegalArgumentException(e);
+                    }
+                  })
+              .collect(toImmutableList());
+      descriptor.getManagedClassNames().clear();
+      descriptor.getManagedClassNames().addAll(nonEntityClasses);
+    }
+
     extraEntityClasses.stream().map(Class::getName).forEach(descriptor::addClasses);
     return Bootstrap.getEntityManagerFactoryBuilder(descriptor, properties).build();
   }

core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java

@@ -101,8 +101,7 @@ public class TransactionManagerTest {
     assertThrows(
         RuntimeException.class,
         () ->
-            tm()
-                .transact(
+            tm().transact(
                     () -> {
                       tm().saveNew(theEntity);
                       throw new RuntimeException();

core/src/test/java/google/registry/rdap/RdapDomainSearchActionTest.java

@@ -37,7 +37,6 @@ import com.google.common.collect.Range;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
-import com.googlecode.objectify.Key;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.Period;
@@ -81,14 +80,18 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
   private HostResource hostNs2CatLol;
   private HashMap<String, HostResource> hostNameToHostMap = new HashMap<>();
 
-  enum RequestType { NONE, NAME, NS_LDH_NAME, NS_IP }
+  enum RequestType {
+    NONE,
+    NAME,
+    NS_LDH_NAME,
+    NS_IP
+  }
 
   private JsonObject generateActualJson(RequestType requestType, String paramValue) {
     return generateActualJson(requestType, paramValue, null);
   }
 
-  private JsonObject generateActualJson(
-      RequestType requestType, String paramValue, String cursor) {
+  private JsonObject generateActualJson(RequestType requestType, String paramValue, String cursor) {
     action.requestPath = actionPath;
     action.requestMethod = POST;
     String requestTypeParam = null;
@@ -179,9 +182,9 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
                 .setCreationClientId("foo")
                 .build());
     persistResource(
-        hostNs1CatLol.asBuilder().setSuperordinateDomain(Key.create(domainCatLol)).build());
+        hostNs1CatLol.asBuilder().setSuperordinateDomain(domainCatLol.createVKey()).build());
     persistResource(
-        hostNs2CatLol.asBuilder().setSuperordinateDomain(Key.create(domainCatLol)).build());
+        hostNs2CatLol.asBuilder().setSuperordinateDomain(domainCatLol.createVKey()).build());
 
     domainCatLol2 =
         persistResource(
@@ -220,8 +223,9 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
                 .build());
     // cat.example
     createTld("example");
-    registrar = persistResource(
-        makeRegistrar("goodregistrar", "St. John Chrysostom", Registrar.State.ACTIVE));
+    registrar =
+        persistResource(
+            makeRegistrar("goodregistrar", "St. John Chrysostom", Registrar.State.ACTIVE));
     persistSimpleResources(makeRegistrarContacts(registrar));
     domainCatExample =
         persistResource(
@@ -297,8 +301,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
                 .build());
     // cat.1.test
     createTld("1.test");
-    registrar =
-        persistResource(makeRegistrar("multiregistrar", "1.test", Registrar.State.ACTIVE));
+    registrar = persistResource(makeRegistrar("multiregistrar", "1.test", Registrar.State.ACTIVE));
     persistSimpleResources(makeRegistrarContacts(registrar));
     domainMultipart =
         persistResource(
@@ -395,10 +398,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
 
   private void deleteCatLol() {
     persistResource(
-        domainCatLol
-            .asBuilder()
-            .setDeletionTime(clock.nowUtc().minusMonths(6))
-            .build());
+        domainCatLol.asBuilder().setDeletionTime(clock.nowUtc().minusMonths(6)).build());
     persistResource(
         makeHistoryEntry(
             domainCatLol,
@@ -416,8 +416,11 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
     for (int i = numHosts; i >= 1; i--) {
       String hostName = String.format("ns%d.%s", i, mainDomainName);
       subordinateHostnamesBuilder.add(hostName);
-      HostResource host = makeAndPersistHostResource(
-          hostName, String.format("5.5.%d.%d", 5 + i / 250, i % 250), clock.nowUtc().minusYears(1));
+      HostResource host =
+          makeAndPersistHostResource(
+              hostName,
+              String.format("5.5.%d.%d", 5 + i / 250, i % 250),
+              clock.nowUtc().minusYears(1));
       hostKeysBuilder.add(host.createVKey());
     }
     ImmutableSet<VKey<HostResource>> hostKeys = hostKeysBuilder.build();
@@ -533,8 +536,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
     assertThat(response.getStatus()).isEqualTo(200);
   }
 
-  private void runNotFoundTest(
-      RequestType requestType, String queryString, String errorMessage) {
+  private void runNotFoundTest(RequestType requestType, String queryString, String errorMessage) {
     rememberWildcardType(queryString);
     assertThat(generateActualJson(requestType, queryString))
         .isEqualTo(generateExpectedJsonError(errorMessage, 404));
@@ -654,9 +656,9 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
   @Test
   public void testInvalidRequest_rejected() {
     assertThat(generateActualJson(RequestType.NONE, null))
-        .isEqualTo(generateExpectedJsonError(
-            "You must specify either name=XXXX, nsLdhName=YYYY or nsIp=ZZZZ",
-            400));
+        .isEqualTo(
+            generateExpectedJsonError(
+                "You must specify either name=XXXX, nsLdhName=YYYY or nsIp=ZZZZ", 400));
     assertThat(response.getStatus()).isEqualTo(400);
     verifyErrorMetrics(SearchType.NONE, Optional.empty(), 400);
   }
@@ -1233,7 +1235,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
     login("evilregistrar");
     runSuccessfulTestWithCatLol(RequestType.NS_LDH_NAME, "ns2.cat.l*", "rdap_domain.json");
     verifyMetrics(SearchType.BY_NAMESERVER_NAME, 1, 1);
- }
+  }
 
   @Test
   public void testNameserverMatchWithWildcard_found_sameRegistrarRequested() {
@@ -1458,24 +1460,21 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
 
   @Test
   public void testNameserverMatchDeletedNameserver_notFound() {
-    persistResource(
-        hostNs1CatLol.asBuilder().setDeletionTime(clock.nowUtc().minusDays(1)).build());
+    persistResource(hostNs1CatLol.asBuilder().setDeletionTime(clock.nowUtc().minusDays(1)).build());
     runNotFoundTest(RequestType.NS_LDH_NAME, "ns1.cat.lol", "No matching nameservers found");
     verifyErrorMetrics(SearchType.BY_NAMESERVER_NAME, Optional.empty(), Optional.of(0L), 404);
   }
 
   @Test
   public void testNameserverMatchDeletedNameserverWithWildcard_notFound() {
-    persistResource(
-        hostNs1CatLol.asBuilder().setDeletionTime(clock.nowUtc().minusDays(1)).build());
+    persistResource(hostNs1CatLol.asBuilder().setDeletionTime(clock.nowUtc().minusDays(1)).build());
     runNotFoundTest(RequestType.NS_LDH_NAME, "ns1.cat.l*", "No matching nameservers found");
     verifyErrorMetrics(SearchType.BY_NAMESERVER_NAME, Optional.empty(), Optional.of(0L), 404);
   }
 
   @Test
   public void testNameserverMatchDeletedNameserverWithWildcardAndSuffix_notFound() {
-    persistResource(
-        hostNs1CatLol.asBuilder().setDeletionTime(clock.nowUtc().minusDays(1)).build());
+    persistResource(hostNs1CatLol.asBuilder().setDeletionTime(clock.nowUtc().minusDays(1)).build());
     runNotFoundTest(RequestType.NS_LDH_NAME, "ns1*.cat.lol", "No matching nameservers found");
     verifyErrorMetrics(SearchType.BY_NAMESERVER_NAME, Optional.empty(), Optional.of(0L), 404);
   }

core/src/test/java/google/registry/rdap/RdapJsonFormatterTest.java

@@ -31,7 +31,6 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
-import com.googlecode.objectify.Key;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.StatusValue;
@@ -159,8 +158,7 @@ public class RdapJsonFormatterTest {
                     "ns1.dog.みんな", null, null, clock.nowUtc().minusYears(6), "unicoderegistrar")
                 .asBuilder()
                 .setSuperordinateDomain(
-                    Key.create(
-                        persistResource(
+                    persistResource(
                             makeDomainBase(
                                     "dog.みんな",
                                     contactResourceRegistrant,
@@ -182,7 +180,8 @@ public class RdapJsonFormatterTest {
                                         .setTransferredRegistrationExpirationTime(
                                             DateTime.parse("2111-10-08T00:44:59Z"))
                                         .build())
-                                .build())))
+                                .build())
+                        .createVKey())
                 .build());
     domainBaseFull =
         persistResource(

core/src/test/java/google/registry/rdap/RdapNameserverSearchActionTest.java

@@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableSet;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.registrar.Registrar;
@@ -111,10 +110,11 @@ public class RdapNameserverSearchActionTest
         persistResource(
             makeRegistrar("evilregistrar", "Yes Virginia <script>", Registrar.State.ACTIVE));
     persistSimpleResources(makeRegistrarContacts(registrar));
-    hostNs1CatLol = makeAndPersistHostResource(
-        "ns1.cat.lol", "1.2.3.4", clock.nowUtc().minusYears(1));
-    hostNs2CatLol = makeAndPersistHostResource(
-        "ns2.cat.lol", "bad:f00d:cafe::15:beef", clock.nowUtc().minusYears(1));
+    hostNs1CatLol =
+        makeAndPersistHostResource("ns1.cat.lol", "1.2.3.4", clock.nowUtc().minusYears(1));
+    hostNs2CatLol =
+        makeAndPersistHostResource(
+            "ns2.cat.lol", "bad:f00d:cafe::15:beef", clock.nowUtc().minusYears(1));
     makeAndPersistHostResource(
         "ns1.cat2.lol", "1.2.3.3", "bad:f00d:cafe::15:beef", clock.nowUtc().minusYears(1));
     makeAndPersistHostResource("ns1.cat.external", null, null, clock.nowUtc().minusYears(1));
@@ -132,25 +132,28 @@ public class RdapNameserverSearchActionTest
     makeAndPersistHostResource("ns1.cat.1.test", "1.2.3.6", clock.nowUtc().minusYears(1));
 
     // create a domain so that we can use it as a test nameserver search string suffix
-    domainCatLol = persistResource(
-        makeDomainBase(
-                "cat.lol",
-                persistResource(
-                    makeContactResource("5372808-ERL", "Goblin Market", "lol@cat.lol", registrar)),
-                persistResource(
-                    makeContactResource("5372808-IRL", "Santa Claus", "BOFH@cat.lol", registrar)),
-                persistResource(
-                    makeContactResource("5372808-TRL", "The Raven", "bog@cat.lol", registrar)),
-                hostNs1CatLol,
-                hostNs2CatLol,
-                registrar)
-            .asBuilder()
-            .setSubordinateHosts(ImmutableSet.of("ns1.cat.lol", "ns2.cat.lol"))
-            .build());
+    domainCatLol =
+        persistResource(
+            makeDomainBase(
+                    "cat.lol",
+                    persistResource(
+                        makeContactResource(
+                            "5372808-ERL", "Goblin Market", "lol@cat.lol", registrar)),
+                    persistResource(
+                        makeContactResource(
+                            "5372808-IRL", "Santa Claus", "BOFH@cat.lol", registrar)),
+                    persistResource(
+                        makeContactResource("5372808-TRL", "The Raven", "bog@cat.lol", registrar)),
+                    hostNs1CatLol,
+                    hostNs2CatLol,
+                    registrar)
+                .asBuilder()
+                .setSubordinateHosts(ImmutableSet.of("ns1.cat.lol", "ns2.cat.lol"))
+                .build());
     persistResource(
-        hostNs1CatLol.asBuilder().setSuperordinateDomain(Key.create(domainCatLol)).build());
+        hostNs1CatLol.asBuilder().setSuperordinateDomain(domainCatLol.createVKey()).build());
     persistResource(
-        hostNs2CatLol.asBuilder().setSuperordinateDomain(Key.create(domainCatLol)).build());
+        hostNs2CatLol.asBuilder().setSuperordinateDomain(domainCatLol.createVKey()).build());
 
     action.ipParam = Optional.empty();
     action.nameParam = Optional.empty();
@@ -187,10 +190,9 @@ public class RdapNameserverSearchActionTest
       hostsBuilder.add(makeHostResource(hostName, "5.5.5.1", "5.5.5.2"));
     }
     persistResources(hostsBuilder.build());
-    domainCatLol = persistResource(
-        domainCatLol.asBuilder()
-            .setSubordinateHosts(subordinateHostsBuilder.build())
-            .build());
+    domainCatLol =
+        persistResource(
+            domainCatLol.asBuilder().setSubordinateHosts(subordinateHostsBuilder.build()).build());
   }
 
   private void createDeletedHost() {
@@ -245,10 +247,7 @@ public class RdapNameserverSearchActionTest
   public void testInvalidRequest_rejected() {
     action.run();
     assertThat(parseJsonObject(response.getPayload()))
-        .isEqualTo(
-            generateExpectedJsonError(
-                "You must specify either name=XXXX or ip=YYYY",
-                400));
+        .isEqualTo(generateExpectedJsonError("You must specify either name=XXXX or ip=YYYY", 400));
     assertThat(response.getStatus()).isEqualTo(400);
     verifyErrorMetrics(Optional.empty(), 400);
   }
@@ -294,8 +293,7 @@ public class RdapNameserverSearchActionTest
   public void testNoCharactersToMatch_rejected() {
     assertThat(generateActualJsonWithName("*"))
         .isEqualTo(
-            generateExpectedJsonError(
-                "Initial search string must be at least 2 characters", 422));
+            generateExpectedJsonError("Initial search string must be at least 2 characters", 422));
     assertThat(response.getStatus()).isEqualTo(422);
     verifyErrorMetrics(Optional.empty(), 422);
   }
@@ -304,8 +302,7 @@ public class RdapNameserverSearchActionTest
   public void testFewerThanTwoCharactersToMatch_rejected() {
     assertThat(generateActualJsonWithName("a*"))
         .isEqualTo(
-            generateExpectedJsonError(
-                "Initial search string must be at least 2 characters", 422));
+            generateExpectedJsonError("Initial search string must be at least 2 characters", 422));
     assertThat(response.getStatus()).isEqualTo(422);
     verifyErrorMetrics(Optional.empty(), 422);
   }
@@ -349,7 +346,8 @@ public class RdapNameserverSearchActionTest
   @Test
   public void testNameMatch_ns2_cat_lol_found() {
     assertThat(generateActualJsonWithName("ns2.cat.lol"))
-        .isEqualTo(generateExpectedJsonForNameserver(
+        .isEqualTo(
+            generateExpectedJsonForNameserver(
                 "ns2.cat.lol",
                 null,
                 "4-ROID",
@@ -505,7 +503,7 @@ public class RdapNameserverSearchActionTest
   public void testNameMatch_nontruncatedResultSet() {
     createManyHosts(4);
     assertThat(generateActualJsonWithName("nsx*.cat.lol"))
-      .isEqualTo(loadJsonFile("rdap_nontruncated_hosts.json"));
+        .isEqualTo(loadJsonFile("rdap_nontruncated_hosts.json"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(4);
   }
@@ -535,8 +533,7 @@ public class RdapNameserverSearchActionTest
 
   @Test
   public void testNameMatchDeletedHost_foundTheOtherHost() {
-    persistResource(
-        hostNs1CatLol.asBuilder().setDeletionTime(clock.nowUtc().minusDays(1)).build());
+    persistResource(hostNs1CatLol.asBuilder().setDeletionTime(clock.nowUtc().minusDays(1)).build());
     assertThat(generateActualJsonWithName("ns*.cat.lol"))
         .isEqualTo(
             generateExpectedJsonForNameserver(
@@ -648,8 +645,7 @@ public class RdapNameserverSearchActionTest
    * @param expectedNames an immutable list of the host names we expect to retrieve
    */
   private void checkCursorNavigation(
-      boolean byName, String paramValue, ImmutableList<String> expectedNames)
-      throws Exception {
+      boolean byName, String paramValue, ImmutableList<String> expectedNames) throws Exception {
     String cursor = null;
     int expectedNameOffset = 0;
     int expectedPageCount =
@@ -792,9 +788,9 @@ public class RdapNameserverSearchActionTest
   public void testAddressMatch_truncatedResultSet() {
     createManyHosts(5);
     assertThat(generateActualJsonWithIp("5.5.5.1"))
-    .isEqualTo(
-        loadJsonFile(
-            "rdap_truncated_hosts.json", "QUERY", "ip=5.5.5.1&cursor=MTctUk9JRA%3D%3D"));
+        .isEqualTo(
+            loadJsonFile(
+                "rdap_truncated_hosts.json", "QUERY", "ip=5.5.5.1&cursor=MTctUk9JRA%3D%3D"));
     assertThat(response.getStatus()).isEqualTo(200);
     verifyMetrics(5, IncompletenessWarningType.TRUNCATED);
   }

core/src/test/java/google/registry/rde/DomainBaseToXjcConverterTest.java

@@ -332,10 +332,9 @@ public class DomainBaseToXjcConverterTest {
                     .setGainingClientId("gaining")
                     .setLosingClientId("losing")
                     .setPendingTransferExpirationTime(DateTime.parse("1925-04-20T00:00:00Z"))
-                    .setServerApproveBillingEvent(Key.create(billingEvent))
+                    .setServerApproveBillingEvent(billingEvent.createVKey())
                     .setServerApproveAutorenewEvent(
-                        Key.create(
-                            persistResource(
+                        persistResource(
                                 new BillingEvent.Recurring.Builder()
                                     .setReason(Reason.RENEW)
                                     .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
@@ -344,10 +343,10 @@ public class DomainBaseToXjcConverterTest {
                                     .setEventTime(END_OF_TIME)
                                     .setRecurrenceEndTime(END_OF_TIME)
                                     .setParent(historyEntry)
-                                    .build())))
+                                    .build())
+                            .createVKey())
                     .setServerApproveAutorenewPollMessage(
-                        Key.create(
-                            persistResource(
+                        persistResource(
                                 new Autorenew.Builder()
                                     .setTargetId("example.xn--q9jyb4c")
                                     .setClientId("TheRegistrar")
@@ -355,8 +354,9 @@ public class DomainBaseToXjcConverterTest {
                                     .setAutorenewEndTime(END_OF_TIME)
                                     .setMsg("Domain was auto-renewed.")
                                     .setParent(historyEntry)
-                                    .build())))
-                    .setServerApproveEntities(ImmutableSet.of(Key.create(billingEvent)))
+                                    .build())
+                            .createVKey())
+                    .setServerApproveEntities(ImmutableSet.of(billingEvent.createVKey()))
                     .setTransferRequestTime(DateTime.parse("1919-01-01T00:00:00Z"))
                     .setTransferStatus(TransferStatus.PENDING)
                     .setTransferredRegistrationExpirationTime(

core/src/test/java/google/registry/rde/HostResourceToXjcConverterTest.java

@@ -23,7 +23,6 @@ import static org.junit.Assert.assertThrows;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.net.InetAddresses;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
@@ -43,8 +42,8 @@ import org.junit.runners.JUnit4;
 /**
  * Unit tests for {@link HostResourceToXjcConverter}.
  *
- * <p>This tests the mapping between {@link HostResource} and {@link XjcRdeHost} as well as
- * some exceptional conditions.
+ * <p>This tests the mapping between {@link HostResource} and {@link XjcRdeHost} as well as some
+ * exceptional conditions.
  */
 @RunWith(JUnit4.class)
 public class HostResourceToXjcConverterTest {
@@ -59,26 +58,29 @@ public class HostResourceToXjcConverterTest {
 
   @Test
   public void testConvertSubordinateHost() {
-    DomainBase domain = newDomainBase("love.foobar").asBuilder()
-        .setPersistedCurrentSponsorClientId("LeisureDog")
-        .setLastTransferTime(DateTime.parse("2010-01-01T00:00:00Z"))
-        .addStatusValue(StatusValue.PENDING_TRANSFER)
-        .build();
-    XjcRdeHost bean = HostResourceToXjcConverter.convertSubordinateHost(
-        new HostResource.Builder()
-            .setCreationClientId("LawyerCat")
-            .setCreationTimeForTest(DateTime.parse("1900-01-01T00:00:00Z"))
-            .setPersistedCurrentSponsorClientId("BusinessCat")
-            .setFullyQualifiedHostName("ns1.love.foobar")
-            .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
-            .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
-            .setLastEppUpdateClientId("CeilingCat")
-            .setLastEppUpdateTime(DateTime.parse("1920-01-01T00:00:00Z"))
-            .setRepoId("2-roid")
-            .setStatusValues(ImmutableSet.of(StatusValue.OK))
-            .setSuperordinateDomain(Key.create(domain))
-            .build(),
-        domain);
+    DomainBase domain =
+        newDomainBase("love.foobar")
+            .asBuilder()
+            .setPersistedCurrentSponsorClientId("LeisureDog")
+            .setLastTransferTime(DateTime.parse("2010-01-01T00:00:00Z"))
+            .addStatusValue(StatusValue.PENDING_TRANSFER)
+            .build();
+    XjcRdeHost bean =
+        HostResourceToXjcConverter.convertSubordinateHost(
+            new HostResource.Builder()
+                .setCreationClientId("LawyerCat")
+                .setCreationTimeForTest(DateTime.parse("1900-01-01T00:00:00Z"))
+                .setPersistedCurrentSponsorClientId("BusinessCat")
+                .setFullyQualifiedHostName("ns1.love.foobar")
+                .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
+                .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
+                .setLastEppUpdateClientId("CeilingCat")
+                .setLastEppUpdateTime(DateTime.parse("1920-01-01T00:00:00Z"))
+                .setRepoId("2-roid")
+                .setStatusValues(ImmutableSet.of(StatusValue.OK))
+                .setSuperordinateDomain(domain.createVKey())
+                .build(),
+            domain);
 
     assertThat(bean.getAddrs()).hasSize(1);
     assertThat(bean.getAddrs().get(0).getIp().value()).isEqualTo("v4");
@@ -119,19 +121,20 @@ public class HostResourceToXjcConverterTest {
 
   @Test
   public void testConvertExternalHost() {
-    XjcRdeHost bean = HostResourceToXjcConverter.convertExternalHost(
-        new HostResource.Builder()
-            .setCreationClientId("LawyerCat")
-            .setCreationTimeForTest(DateTime.parse("1900-01-01T00:00:00Z"))
-            .setPersistedCurrentSponsorClientId("BusinessCat")
-            .setFullyQualifiedHostName("ns1.love.lol")
-            .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
-            .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
-            .setLastEppUpdateClientId("CeilingCat")
-            .setLastEppUpdateTime(DateTime.parse("1920-01-01T00:00:00Z"))
-            .setRepoId("2-roid")
-            .setStatusValues(ImmutableSet.of(StatusValue.OK))
-            .build());
+    XjcRdeHost bean =
+        HostResourceToXjcConverter.convertExternalHost(
+            new HostResource.Builder()
+                .setCreationClientId("LawyerCat")
+                .setCreationTimeForTest(DateTime.parse("1900-01-01T00:00:00Z"))
+                .setPersistedCurrentSponsorClientId("BusinessCat")
+                .setFullyQualifiedHostName("ns1.love.lol")
+                .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
+                .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
+                .setLastEppUpdateClientId("CeilingCat")
+                .setLastEppUpdateTime(DateTime.parse("1920-01-01T00:00:00Z"))
+                .setRepoId("2-roid")
+                .setStatusValues(ImmutableSet.of(StatusValue.OK))
+                .build());
 
     assertThat(bean.getAddrs()).hasSize(1);
     assertThat(bean.getAddrs().get(0).getIp().value()).isEqualTo("v4");
@@ -167,19 +170,20 @@ public class HostResourceToXjcConverterTest {
 
   @Test
   public void testConvertExternalHost_ipv6() {
-    XjcRdeHost bean = HostResourceToXjcConverter.convertExternalHost(
-        new HostResource.Builder()
-            .setCreationClientId("LawyerCat")
-            .setCreationTimeForTest(DateTime.parse("1900-01-01T00:00:00Z"))
-            .setPersistedCurrentSponsorClientId("BusinessCat")
-            .setFullyQualifiedHostName("ns1.love.lol")
-            .setInetAddresses(ImmutableSet.of(InetAddresses.forString("cafe::abba")))
-            .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
-            .setLastEppUpdateClientId("CeilingCat")
-            .setLastEppUpdateTime(DateTime.parse("1920-01-01T00:00:00Z"))
-            .setRepoId("2-LOL")
-            .setStatusValues(ImmutableSet.of(StatusValue.OK))
-            .build());
+    XjcRdeHost bean =
+        HostResourceToXjcConverter.convertExternalHost(
+            new HostResource.Builder()
+                .setCreationClientId("LawyerCat")
+                .setCreationTimeForTest(DateTime.parse("1900-01-01T00:00:00Z"))
+                .setPersistedCurrentSponsorClientId("BusinessCat")
+                .setFullyQualifiedHostName("ns1.love.lol")
+                .setInetAddresses(ImmutableSet.of(InetAddresses.forString("cafe::abba")))
+                .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
+                .setLastEppUpdateClientId("CeilingCat")
+                .setLastEppUpdateTime(DateTime.parse("1920-01-01T00:00:00Z"))
+                .setRepoId("2-LOL")
+                .setStatusValues(ImmutableSet.of(StatusValue.OK))
+                .build());
     assertThat(bean.getAddrs()).hasSize(1);
     assertThat(bean.getAddrs().get(0).getIp().value()).isEqualTo("v6");
     assertThat(bean.getAddrs().get(0).getValue()).isEqualTo("cafe::abba");
@@ -208,19 +212,20 @@ public class HostResourceToXjcConverterTest {
   @Test
   public void testMarshal() throws Exception {
     // Bean! Bean! Bean!
-    XjcRdeHostElement bean = HostResourceToXjcConverter.convertExternal(
-        new HostResource.Builder()
-            .setCreationClientId("LawyerCat")
-            .setCreationTimeForTest(DateTime.parse("1900-01-01T00:00:00Z"))
-            .setPersistedCurrentSponsorClientId("BusinessCat")
-            .setFullyQualifiedHostName("ns1.love.lol")
-            .setInetAddresses(ImmutableSet.of(InetAddresses.forString("cafe::abba")))
-            .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
-            .setLastEppUpdateClientId("CeilingCat")
-            .setLastEppUpdateTime(DateTime.parse("1920-01-01T00:00:00Z"))
-            .setRepoId("2-LOL")
-            .setStatusValues(ImmutableSet.of(StatusValue.OK))
-            .build());
+    XjcRdeHostElement bean =
+        HostResourceToXjcConverter.convertExternal(
+            new HostResource.Builder()
+                .setCreationClientId("LawyerCat")
+                .setCreationTimeForTest(DateTime.parse("1900-01-01T00:00:00Z"))
+                .setPersistedCurrentSponsorClientId("BusinessCat")
+                .setFullyQualifiedHostName("ns1.love.lol")
+                .setInetAddresses(ImmutableSet.of(InetAddresses.forString("cafe::abba")))
+                .setLastTransferTime(DateTime.parse("1910-01-01T00:00:00Z"))
+                .setLastEppUpdateClientId("CeilingCat")
+                .setLastEppUpdateTime(DateTime.parse("1920-01-01T00:00:00Z"))
+                .setRepoId("2-LOL")
+                .setStatusValues(ImmutableSet.of(StatusValue.OK))
+                .build());
     marshalStrict(bean, new ByteArrayOutputStream(), UTF_8);
   }
 }

core/src/test/java/google/registry/rde/RdeFixtures.java

@@ -174,10 +174,9 @@ final class RdeFixtures {
                     .setGainingClientId("gaining")
                     .setLosingClientId("losing")
                     .setPendingTransferExpirationTime(DateTime.parse("1993-04-20T00:00:00Z"))
-                    .setServerApproveBillingEvent(Key.create(billingEvent))
+                    .setServerApproveBillingEvent(billingEvent.createVKey())
                     .setServerApproveAutorenewEvent(
-                        Key.create(
-                            persistResource(
+                        persistResource(
                                 new BillingEvent.Recurring.Builder()
                                     .setReason(Reason.RENEW)
                                     .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
@@ -186,10 +185,10 @@ final class RdeFixtures {
                                     .setEventTime(END_OF_TIME)
                                     .setRecurrenceEndTime(END_OF_TIME)
                                     .setParent(historyEntry)
-                                    .build())))
+                                    .build())
+                            .createVKey())
                     .setServerApproveAutorenewPollMessage(
-                        Key.create(
-                            persistResource(
+                        persistResource(
                                 new Autorenew.Builder()
                                     .setTargetId("example." + tld)
                                     .setClientId("TheRegistrar")
@@ -197,8 +196,9 @@ final class RdeFixtures {
                                     .setAutorenewEndTime(END_OF_TIME)
                                     .setMsg("Domain was auto-renewed.")
                                     .setParent(historyEntry)
-                                    .build())))
-                    .setServerApproveEntities(ImmutableSet.of(Key.create(billingEvent)))
+                                    .build())
+                            .createVKey())
+                    .setServerApproveEntities(ImmutableSet.of(billingEvent.createVKey()))
                     .setTransferRequestTime(DateTime.parse("1991-01-01T00:00:00Z"))
                     .setTransferStatus(TransferStatus.PENDING)
                     .setTransferredRegistrationExpirationTime(

core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java

@@ -77,6 +77,26 @@ public class Spec11RegistrarThreatMatchesParserTest {
         .hasValue(LocalDate.parse("2018-07-14"));
   }
 
+  @Test
+  public void testSuccess_ignoreExtraFields() throws Exception {
+    ThreatMatch objectWithExtraFields =
+        ThreatMatch.fromJSON(
+            new JSONObject(
+                ImmutableMap.of(
+                    "threatType", "MALWARE",
+                    "platformType", "ANY_PLATFORM",
+                    "threatEntryMetaData", "NONE",
+                    "fullyQualifiedDomainName", "c.com")));
+    ThreatMatch objectWithoutExtraFields =
+        ThreatMatch.fromJSON(
+            new JSONObject(
+                ImmutableMap.of(
+                    "threatType", "MALWARE",
+                    "fullyQualifiedDomainName", "c.com")));
+
+    assertThat(objectWithExtraFields).isEqualTo(objectWithoutExtraFields);
+  }
+
   /** The expected contents of the sample spec11 report file */
   static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
     return ImmutableSet.of(getMatchA(), getMatchB());
@@ -90,8 +110,6 @@ public class Spec11RegistrarThreatMatchesParserTest {
                 new JSONObject(
                     ImmutableMap.of(
                         "threatType", "MALWARE",
-                        "platformType", "ANY_PLATFORM",
-                        "threatEntryMetadata", "NONE",
                         "fullyQualifiedDomainName", "a.com")))));
   }
 
@@ -103,15 +121,11 @@ public class Spec11RegistrarThreatMatchesParserTest {
                 new JSONObject(
                     ImmutableMap.of(
                         "threatType", "MALWARE",
-                        "platformType", "ANY_PLATFORM",
-                        "threatEntryMetadata", "NONE",
                         "fullyQualifiedDomainName", "b.com"))),
             ThreatMatch.fromJSON(
                 new JSONObject(
                     ImmutableMap.of(
                         "threatType", "MALWARE",
-                        "platformType", "ANY_PLATFORM",
-                        "threatEntryMetadata", "NONE",
                         "fullyQualifiedDomainName", "c.com")))));
   }
 

core/src/test/java/google/registry/testing/AbstractEppResourceSubject.java

@@ -185,7 +185,7 @@ abstract class AbstractEppResourceSubject<
     return andChainer();
   }
 
-  protected <E> And<S> hasValue(E expected, E actual, String name) {
+  protected <E> And<S> hasValue(@Nullable E expected, @Nullable E actual, String name) {
     check(name).that(actual).isEqualTo(expected);
     return andChainer();
   }