Only use asymmetric VKeys for EPP resources (#611)

Given that we currently have no way of reconstituting a symmetric key
from the asymmetric key (or at least, we don't have a 100% reliable way
of doing so) it's best to keep keys as asymmetrical, referring to the
correct database. That way, we don't get situations where we cannot
compare equality of two keys due to one being asymmetrical and one being
symmetrical.

build.gradle

@@ -458,6 +458,7 @@ task javadoc(type: Javadoc) {
   source javadocSource
   classpath = files(javadocClasspath)
   destinationDir = file("${buildDir}/docs/javadoc")
+  options.encoding = "UTF-8"
   // In a lot of places we don't write @return so suppress warnings about that.
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
@@ -467,4 +468,4 @@ task javadoc(type: Javadoc) {
 
 tasks.build.dependsOn(tasks.javadoc)
 
-javadocDependentTasks.each { tasks.javadoc.shouldRunAfter(it) }
+javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -150,7 +150,7 @@ public final class AsyncTaskEnqueuer {
 
   /** Enqueues a task to asynchronously refresh DNS for a renamed host. */
   public void enqueueAsyncDnsRefresh(HostResource host, DateTime now) {
-    VKey<HostResource> hostKey = host.createKey();
+    VKey<HostResource> hostKey = host.createVKey();
     logger.atInfo().log("Enqueuing async DNS refresh for renamed host %s.", hostKey);
     addTaskToQueueWithRetry(
         asyncDnsRefreshPullQueue,

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -0,0 +1,86 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a
+ * Datastore export and Nomulus CommitLogs that overlap with the export.
+ */
+public final class BackupPaths {
+
+  private BackupPaths() {}
+
+  private static final String WILDCARD_CHAR = "*";
+  private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/input-%s";
+  /**
+   * Regex pattern that captures the kind string in a file name. Datastore places no restrictions on
+   * what characters may be used in a kind string.
+   */
+  private static final String FILENAME_TO_KIND_REGEX = ".+/all_namespaces/kind_(.+)/input-.+";
+
+  private static final Pattern FILENAME_TO_KIND_PATTERN = Pattern.compile(FILENAME_TO_KIND_REGEX);
+
+  /**
+   * Returns a regex pattern that matches all Datastore export files of a given {@code kind}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kind the 'kind' of the Datastore entity
+   */
+  public static String getExportFileNamePattern(String exportDir, String kind) {
+    checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
+    checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, WILDCARD_CHAR);
+  }
+
+  /**
+   * Returns the fully qualified path of a Datastore export file with the given {@code kind} and
+   * {@code shard}.
+   *
+   * @param exportDir path to the top directory of a Datastore export
+   * @param kind the 'kind' of the Datastore entity
+   * @param shard an integer suffix of the file name
+   */
+  public static String getExportFileNameByShard(String exportDir, String kind, int shard) {
+    checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
+    checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
+    checkArgument(shard >= 0, "Negative shard %s not allowed.", shard);
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, Integer.toString(shard));
+  }
+
+  /**
+   * Returns the 'kind' of entity stored in a file based on the file name.
+   *
+   * <p>This method poses low risk and greatly simplifies the implementation of some transforms in
+   * {@link ExportLoadingTransforms}.
+   *
+   * @see ExportLoadingTransforms
+   */
+  public static String getKindFromFileName(String fileName) {
+    checkArgument(!isNullOrEmpty(fileName), "Null or empty fileName.");
+    Matcher matcher = FILENAME_TO_KIND_PATTERN.matcher(fileName);
+    checkArgument(
+        matcher.matches(),
+        "Illegal file name %s, should match %s.",
+        fileName,
+        FILENAME_TO_KIND_REGEX);
+    return matcher.group(1);
+  }
+}

core/src/main/java/google/registry/beam/initsql/ExportLoadingTransforms.java

@@ -0,0 +1,117 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static google.registry.beam.initsql.BackupPaths.getExportFileNamePattern;
+import static google.registry.beam.initsql.BackupPaths.getKindFromFileName;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.collect.ImmutableList;
+import google.registry.tools.LevelDbLogReader;
+import java.io.IOException;
+import java.util.Collection;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.fs.EmptyMatchTreatment;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+/**
+ * {@link PTransform Pipeline transforms} for loading from Datastore export files. They are all part
+ * of a transformation that loads raw records from a Datastore export, and are broken apart for
+ * testing.
+ *
+ * <p>We drop the 'kind' information in {@link #getDatastoreExportFilePatterns} and recover it later
+ * using the file paths. Although we could have kept it by passing around {@link KV key-value
+ * pairs}, the code would be more complicated, especially in {@link #loadDataFromFiles()}.
+ */
+public class ExportLoadingTransforms {
+
+  /**
+   * Returns a {@link PTransform transform} that can generate a collection of patterns that match
+   * all Datastore export files of the given {@code kinds}.
+   */
+  public static PTransform<PBegin, PCollection<String>> getDatastoreExportFilePatterns(
+      String exportDir, Collection<String> kinds) {
+    return Create.of(
+            kinds.stream()
+                .map(kind -> getExportFileNamePattern(exportDir, kind))
+                .collect(ImmutableList.toImmutableList()))
+        .withCoder(StringUtf8Coder.of());
+  }
+
+  /**
+   * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}.
+   */
+  public static PTransform<PCollection<String>, PCollection<Metadata>> getFilesByPatterns() {
+    return new PTransform<PCollection<String>, PCollection<Metadata>>() {
+      @Override
+      public PCollection<Metadata> expand(PCollection<String> input) {
+        return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.DISALLOW));
+      }
+    };
+  }
+
+  /**
+   * Returns a {@link PTransform} from file {@link Metadata} to {@link KV Key-Value pairs} with
+   * entity 'kind' as key and raw record as value.
+   */
+  public static PTransform<PCollection<Metadata>, PCollection<KV<String, byte[]>>>
+      loadDataFromFiles() {
+    return new PTransform<PCollection<Metadata>, PCollection<KV<String, byte[]>>>() {
+      @Override
+      public PCollection<KV<String, byte[]>> expand(PCollection<Metadata> input) {
+        return input
+            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+            .apply(
+                MapElements.into(kvs(strings(), TypeDescriptor.of(ReadableFile.class)))
+                    .via(file -> KV.of(getKindFromFileName(file.getMetadata().toString()), file)))
+            .apply("Load One LevelDb File", ParDo.of(new LoadOneFile()));
+      }
+    };
+  }
+
+  /**
+   * Reads a LevelDb file and converts each raw record into a {@link KV pair} of kind and bytes.
+   *
+   * <p>LevelDb files are not seekable because a large object may span multiple blocks. If a
+   * sequential read fails, the file needs to be retried from the beginning.
+   */
+  private static class LoadOneFile extends DoFn<KV<String, ReadableFile>, KV<String, byte[]>> {
+
+    @ProcessElement
+    public void processElement(
+        @Element KV<String, ReadableFile> kv, OutputReceiver<KV<String, byte[]>> output) {
+      try {
+        LevelDbLogReader.from(kv.getValue().open())
+            .forEachRemaining(record -> output.output(KV.of(kv.getKey(), record)));
+      } catch (IOException e) {
+        // Let the pipeline retry the whole file.
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

core/src/main/java/google/registry/beam/initsql/README.md

@@ -0,0 +1,3 @@
+## Summary
+
+This package contains a BEAM pipeline that populates a Cloud SQL database from a Datastore backup.

core/src/main/java/google/registry/beam/spec11/ThreatMatch.java

@@ -24,22 +24,10 @@ import org.json.JSONObject;
 public abstract class ThreatMatch implements Serializable {
 
   private static final String THREAT_TYPE_FIELD = "threatType";
-  private static final String PLATFORM_TYPE_FIELD = "platformType";
-  private static final String METADATA_FIELD = "threatEntryMetadata";
   private static final String DOMAIN_NAME_FIELD = "fullyQualifiedDomainName";
 
   /** Returns what kind of threat it is (malware, phishing etc.) */
   public abstract String threatType();
-  /** Returns what platforms it affects (Windows, Linux etc.) */
-  abstract String platformType();
-  /**
-   * Returns a String representing a JSON Object containing arbitrary metadata associated with this
-   * threat, or "NONE" if there is no metadata to retrieve.
-   *
-   * <p>This ideally would be a {@link JSONObject} type, but can't be due to serialization
-   * requirements.
-   */
-  abstract String metadata();
   /** Returns the fully qualified domain name [SLD].[TLD] of the matched threat. */
   public abstract String fullyQualifiedDomainName();
 
@@ -52,29 +40,19 @@ public abstract class ThreatMatch implements Serializable {
   static ThreatMatch create(JSONObject threatMatchJSON, String fullyQualifiedDomainName)
       throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatchJSON.getString(THREAT_TYPE_FIELD),
-        threatMatchJSON.getString(PLATFORM_TYPE_FIELD),
-        threatMatchJSON.has(METADATA_FIELD)
-            ? threatMatchJSON.getJSONObject(METADATA_FIELD).toString()
-            : "NONE",
-        fullyQualifiedDomainName);
+        threatMatchJSON.getString(THREAT_TYPE_FIELD), fullyQualifiedDomainName);
   }
 
   /** Returns a {@link JSONObject} representing a subset of this object's data. */
   JSONObject toJSON() throws JSONException {
     return new JSONObject()
         .put(THREAT_TYPE_FIELD, threatType())
-        .put(PLATFORM_TYPE_FIELD, platformType())
-        .put(METADATA_FIELD, metadata())
         .put(DOMAIN_NAME_FIELD, fullyQualifiedDomainName());
   }
 
   /** Parses a {@link JSONObject} and returns an equivalent {@link ThreatMatch}. */
   public static ThreatMatch fromJSON(JSONObject threatMatch) throws JSONException {
     return new AutoValue_ThreatMatch(
-        threatMatch.getString(THREAT_TYPE_FIELD),
-        threatMatch.getString(PLATFORM_TYPE_FIELD),
-        threatMatch.getString(METADATA_FIELD),
-        threatMatch.getString(DOMAIN_NAME_FIELD));
+        threatMatch.getString(THREAT_TYPE_FIELD), threatMatch.getString(DOMAIN_NAME_FIELD));
   }
 }

core/src/main/java/google/registry/flows/ResourceFlowUtils.java

@@ -94,10 +94,10 @@ public final class ResourceFlowUtils {
                    * trust the query and need to do the full mapreduce.
                    */
                   Iterable<Key<DomainBase>> keys =
-                      queryForLinkedDomains(fki.getResourceKey(), now)
+                      queryForLinkedDomains(fki.getResourceKey().getOfyKey(), now)
                           .limit(FAILFAST_CHECK_COUNT)
                           .keys();
-                  VKey<R> resourceVKey = VKey.createOfy(resourceClass, fki.getResourceKey());
+                  VKey<R> resourceVKey = fki.getResourceKey();
                   Predicate<DomainBase> predicate =
                       domain -> getPotentialReferences.apply(domain).contains(resourceVKey);
                   return ofy().load().keys(keys).values().stream().anyMatch(predicate)
@@ -136,9 +136,9 @@ public final class ResourceFlowUtils {
 
   public static <R extends EppResource> void verifyResourceDoesNotExist(
       Class<R> clazz, String targetId, DateTime now, String clientId) throws EppException {
-    Key<R> key = loadAndGetKey(clazz, targetId, now);
+    VKey<R> key = loadAndGetKey(clazz, targetId, now);
     if (key != null) {
-      R resource = ofy().load().key(key).now();
+      R resource = tm().load(key);
       // These are similar exceptions, but we can track them internally as log-based metrics.
       if (Objects.equals(clientId, resource.getPersistedCurrentSponsorClientId())) {
         throw new ResourceAlreadyExistsForThisClientException(targetId);

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -77,7 +77,6 @@ import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.billing.BillingEvent.Recurring;
-import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainCommand;
 import google.registry.model.domain.DomainCommand.Create;
@@ -353,14 +352,12 @@ public class DomainCreateFlow implements TransactionalFlow {
             .setLaunchNotice(hasClaimsNotice ? launchCreate.get().getNotice() : null)
             .setSmdId(signedMarkId)
             .setDsData(secDnsCreate.isPresent() ? secDnsCreate.get().getDsData() : null)
-            .setRegistrant(VKey.createOfy(ContactResource.class, command.getRegistrant()))
+            .setRegistrant(command.getRegistrant())
             .setAuthInfo(command.getAuthInfo())
             .setFullyQualifiedDomainName(targetId)
             .setNameservers(
                 (ImmutableSet<VKey<HostResource>>)
-                    command.getNameservers().stream()
-                        .map(key -> VKey.createOfy(HostResource.class, key))
-                        .collect(toImmutableSet()))
+                    command.getNameservers().stream().collect(toImmutableSet()))
             .setStatusValues(statuses.build())
             .setContacts(command.getContacts())
             .addGracePeriod(GracePeriod.forBillingEvent(GracePeriodStatus.ADD, createBillingEvent))

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -307,14 +307,11 @@ public class DomainFlowUtils {
   /** Verify that no linked resources have disallowed statuses. */
   static void verifyNotInPendingDelete(
       Set<DesignatedContact> contacts,
-      Key<ContactResource> registrant,
-      Set<Key<HostResource>> nameservers)
+      VKey<ContactResource> registrant,
+      Set<VKey<HostResource>> nameservers)
       throws EppException {
-    ImmutableList.Builder<Key<? extends EppResource>> keysToLoad = new ImmutableList.Builder<>();
-    contacts.stream()
-        .map(DesignatedContact::getContactKey)
-        .map(VKey::getOfyKey)
-        .forEach(keysToLoad::add);
+    ImmutableList.Builder<VKey<? extends EppResource>> keysToLoad = new ImmutableList.Builder<>();
+    contacts.stream().map(DesignatedContact::getContactKey).forEach(keysToLoad::add);
     Optional.ofNullable(registrant).ifPresent(keysToLoad::add);
     keysToLoad.addAll(nameservers);
     verifyNotInPendingDelete(EppResource.loadCached(keysToLoad.build()).values());
@@ -381,7 +378,7 @@ public class DomainFlowUtils {
   }
 
   static void validateRequiredContactsPresent(
-      @Nullable Key<ContactResource> registrant, Set<DesignatedContact> contacts)
+      @Nullable VKey<ContactResource> registrant, Set<DesignatedContact> contacts)
       throws RequiredParameterMissingException {
     if (registrant == null) {
       throw new MissingRegistrantException();

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -60,7 +60,6 @@ import google.registry.flows.domain.DomainFlowUtils.MissingRegistrantException;
 import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
-import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainCommand.Update;
 import google.registry.model.domain.DomainCommand.Update.AddRemove;
@@ -73,11 +72,9 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
-import google.registry.model.host.HostResource;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.persistence.VKey;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
@@ -247,22 +244,11 @@ public final class DomainUpdateFlow implements TransactionalFlow {
             .setLastEppUpdateClientId(clientId)
             .addStatusValues(add.getStatusValues())
             .removeStatusValues(remove.getStatusValues())
-            .addNameservers(
-                add.getNameservers().stream()
-                    .map(key -> VKey.createOfy(HostResource.class, key))
-                    .collect(toImmutableSet()))
-            .removeNameservers(
-                remove.getNameservers().stream()
-                    .map(key -> VKey.createOfy(HostResource.class, key))
-                    .collect(toImmutableSet()))
+            .addNameservers(add.getNameservers().stream().collect(toImmutableSet()))
+            .removeNameservers(remove.getNameservers().stream().collect(toImmutableSet()))
             .addContacts(add.getContacts())
             .removeContacts(remove.getContacts())
-            .setRegistrant(
-                firstNonNull(
-                    change.getRegistrant() != null
-                        ? VKey.createOfy(ContactResource.class, change.getRegistrant())
-                        : null,
-                    domain.getRegistrant()))
+            .setRegistrant(firstNonNull(change.getRegistrant(), domain.getRegistrant()))
             .setAuthInfo(firstNonNull(change.getAuthInfo(), domain.getAuthInfo()));
     return domainBuilder.build();
   }
@@ -275,7 +261,7 @@ public final class DomainUpdateFlow implements TransactionalFlow {
 
   private void validateNewState(DomainBase newDomain) throws EppException {
     validateNoDuplicateContacts(newDomain.getContacts());
-    validateRequiredContactsPresent(newDomain.getRegistrant().getOfyKey(), newDomain.getContacts());
+    validateRequiredContactsPresent(newDomain.getRegistrant(), newDomain.getContacts());
     validateDsData(newDomain.getDsData());
     validateNameserversCountForTld(
         newDomain.getTld(),

core/src/main/java/google/registry/model/EppResource.java

@@ -16,12 +16,11 @@ package google.registry.model;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
@@ -32,11 +31,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Index;
@@ -44,12 +41,13 @@ import google.registry.config.RegistryConfig;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.transfer.TransferData;
+import google.registry.persistence.VKey;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.StreamSupport;
 import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
@@ -183,6 +181,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   /** Get the foreign key string for this resource. */
   public abstract String getForeignKey();
 
+  /** Create the VKey for the specified EPP resource. */
+  public abstract VKey<? extends EppResource> createVKey();
+
   /** Override of {@link Buildable#asBuilder} so that the extra methods are visible. */
   @Override
   public abstract Builder<?, ?> asBuilder();
@@ -329,18 +330,18 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     }
   }
 
-  static final CacheLoader<Key<? extends EppResource>, EppResource> CACHE_LOADER =
-      new CacheLoader<Key<? extends EppResource>, EppResource>() {
+  static final CacheLoader<VKey<? extends EppResource>, EppResource> CACHE_LOADER =
+      new CacheLoader<VKey<? extends EppResource>, EppResource>() {
 
         @Override
-        public EppResource load(Key<? extends EppResource> key) {
-          return tm().doTransactionless(() -> ofy().load().key(key).now());
+        public EppResource load(VKey<? extends EppResource> key) {
+          return tm().doTransactionless(() -> tm().load(key));
         }
 
         @Override
-        public Map<Key<? extends EppResource>, EppResource> loadAll(
-            Iterable<? extends Key<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> loadMultiple(keys));
+        public Map<VKey<? extends EppResource>, EppResource> loadAll(
+            Iterable<? extends VKey<? extends EppResource>> keys) {
+          return tm().doTransactionless(() -> loadAsMap(keys));
         }
       };
 
@@ -352,10 +353,10 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * directly should of course never use the cache.
    */
   @NonFinalForTesting
-  private static LoadingCache<Key<? extends EppResource>, EppResource> cacheEppResources =
+  private static LoadingCache<VKey<? extends EppResource>, EppResource> cacheEppResources =
       createEppResourcesCache(getEppResourceCachingDuration());
 
-  private static LoadingCache<Key<? extends EppResource>, EppResource> createEppResourcesCache(
+  private static LoadingCache<VKey<? extends EppResource>, EppResource> createEppResourcesCache(
       Duration expiry) {
     return CacheBuilder.newBuilder()
         .expireAfterWrite(expiry.getMillis(), MILLISECONDS)
@@ -369,29 +370,16 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     cacheEppResources = createEppResourcesCache(effectiveExpiry);
   }
 
-  private static ImmutableMap<Key<? extends EppResource>, EppResource> loadMultiple(
-      Iterable<? extends Key<? extends EppResource>> keys) {
-    // This cast is safe because, in Objectify, Key<? extends EppResource> can also be
-    // treated as a Key<EppResource>.
-    @SuppressWarnings("unchecked")
-    ImmutableList<Key<EppResource>> typedKeys =
-        Streams.stream(keys).map(key -> (Key<EppResource>) key).collect(toImmutableList());
-
-    // Typing shenanigans are required to return the map with the correct required generic type.
-    return ofy().load().keys(typedKeys).entrySet().stream()
-        .collect(ImmutableMap.toImmutableMap(Entry::getKey, Entry::getValue));
-  }
-
   /**
    * Loads the given EppResources by their keys using the cache (if enabled).
    *
    * <p>Don't use this unless you really need it for performance reasons, and be sure that you are
    * OK with the trade-offs in loss of transactional consistency.
    */
-  public static ImmutableMap<Key<? extends EppResource>, EppResource> loadCached(
-      Iterable<Key<? extends EppResource>> keys) {
+  public static ImmutableMap<VKey<? extends EppResource>, EppResource> loadCached(
+      Iterable<VKey<? extends EppResource>> keys) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return loadMultiple(keys);
+      return loadAsMap(keys);
     }
     try {
       return cacheEppResources.getAll(keys);
@@ -406,9 +394,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>Don't use this unless you really need it for performance reasons, and be sure that you are
    * OK with the trade-offs in loss of transactional consistency.
    */
-  public static <T extends EppResource> T loadCached(Key<T> key) {
+  public static <T extends EppResource> T loadCached(VKey<T> key) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
-      return ofy().load().key(key).now();
+      return tm().load(key);
     }
     try {
       // Safe to cast because loading a Key<T> returns an entity of type T.
@@ -419,4 +407,17 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
       throw new RuntimeException("Error loading cached EppResources", e.getCause());
     }
   }
+
+  private static ImmutableMap<VKey<? extends EppResource>, EppResource> loadAsMap(
+      Iterable<? extends VKey<? extends EppResource>> keys) {
+    return StreamSupport.stream(keys.spliterator(), false)
+        // It's possible for us to receive the same key more than once which causes
+        // the immutable map build to break with a duplicate key, so we have to ensure key
+        // uniqueness.
+        .distinct()
+        // We have to use "key -> key" here instead of the identity() function, because
+        // the latter breaks the fairly complicated generic type checking required by the
+        // caching interface.
+        .collect(toImmutableMap(key -> key, key -> tm().load(key)));
+  }
 }

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -17,6 +17,7 @@ package google.registry.model;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;
@@ -141,7 +142,7 @@ public final class EppResourceUtils {
     T resource =
         useCache
             ? EppResource.loadCached(fki.getResourceKey())
-            : ofy().load().key(fki.getResourceKey()).now();
+            : tm().maybeLoad(fki.getResourceKey()).orElse(null);
     if (resource == null || isAtOrAfter(now, resource.getDeletionTime())) {
       return Optional.empty();
     }

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -197,7 +197,9 @@ public class ContactResource extends EppResource
   })
   Disclose disclose;
 
+  @Override
   public VKey<ContactResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.createOfy(ContactResource.class, Key.create(this));
   }
 

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -613,6 +613,12 @@ public class DomainBase extends EppResource
     }
   }
 
+  @Override
+  public VKey<DomainBase> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
+    return VKey.createOfy(DomainBase.class, Key.create(this));
+  }
+
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
   private static final Predicate<DesignatedContact> IS_REGISTRANT =
       (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);

core/src/main/java/google/registry/model/domain/DomainCommand.java

@@ -30,7 +30,6 @@ import com.google.common.base.MoreObjects;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.contact.ContactResource;
@@ -82,8 +81,7 @@ public class DomainCommand {
     String registrantContactId;
 
     /** A resolved key to the registrant who registered this domain. */
-    @XmlTransient
-    Key<ContactResource> registrant;
+    @XmlTransient VKey<ContactResource> registrant;
 
     /** Authorization info (aka transfer secret) of the domain. */
     DomainAuthInfo authInfo;
@@ -93,7 +91,7 @@ public class DomainCommand {
     }
 
     @Nullable
-    public Key<ContactResource> getRegistrant() {
+    public VKey<ContactResource> getRegistrant() {
       return registrant;
     }
 
@@ -129,8 +127,7 @@ public class DomainCommand {
     Set<String> nameserverFullyQualifiedHostNames;
 
     /** Resolved keys to hosts that are the nameservers for the domain. */
-    @XmlTransient
-    Set<Key<HostResource>> nameservers;
+    @XmlTransient Set<VKey<HostResource>> nameservers;
 
     /** Foreign keyed associated contacts for the domain (other than registrant). */
     @XmlElement(name = "contact")
@@ -160,7 +157,7 @@ public class DomainCommand {
       return nullToEmptyImmutableCopy(nameserverFullyQualifiedHostNames);
     }
 
-    public ImmutableSet<Key<HostResource>> getNameservers() {
+    public ImmutableSet<VKey<HostResource>> getNameservers() {
       return nullToEmptyImmutableCopy(nameservers);
     }
 
@@ -190,7 +187,7 @@ public class DomainCommand {
             now);
         for (DesignatedContact contact : contacts) {
           if (DesignatedContact.Type.REGISTRANT.equals(contact.getType())) {
-            clone.registrant = contact.getContactKey().getOfyKey();
+            clone.registrant = contact.getContactKey();
             clone.contacts = forceEmptyToNull(difference(contacts, contact));
             break;
           }
@@ -354,8 +351,7 @@ public class DomainCommand {
       Set<String> nameserverFullyQualifiedHostNames;
 
       /** Resolved keys to hosts that are the nameservers for the domain. */
-      @XmlTransient
-      Set<Key<HostResource>> nameservers;
+      @XmlTransient Set<VKey<HostResource>> nameservers;
 
       /** Foreign keyed associated contacts for the domain (other than registrant). */
       @XmlElement(name = "contact")
@@ -369,7 +365,7 @@ public class DomainCommand {
         return nullSafeImmutableCopy(nameserverFullyQualifiedHostNames);
       }
 
-      public ImmutableSet<Key<HostResource>> getNameservers() {
+      public ImmutableSet<VKey<HostResource>> getNameservers() {
         return nullToEmptyImmutableCopy(nameservers);
       }
 
@@ -419,7 +415,7 @@ public class DomainCommand {
     }
   }
 
-  private static Set<Key<HostResource>> linkHosts(
+  private static Set<VKey<HostResource>> linkHosts(
       Set<String> fullyQualifiedHostNames, DateTime now) throws InvalidReferencesException {
     if (fullyQualifiedHostNames == null) {
       return null;
@@ -437,20 +433,18 @@ public class DomainCommand {
     for (ForeignKeyedDesignatedContact contact : contacts) {
       foreignKeys.add(contact.contactId);
     }
-    ImmutableMap<String, Key<ContactResource>> loadedContacts =
+    ImmutableMap<String, VKey<ContactResource>> loadedContacts =
         loadByForeignKeysCached(foreignKeys.build(), ContactResource.class, now);
     ImmutableSet.Builder<DesignatedContact> linkedContacts = new ImmutableSet.Builder<>();
     for (ForeignKeyedDesignatedContact contact : contacts) {
       linkedContacts.add(
-          DesignatedContact.create(
-              contact.type,
-              VKey.createOfy(ContactResource.class, loadedContacts.get(contact.contactId))));
+          DesignatedContact.create(contact.type, loadedContacts.get(contact.contactId)));
     }
     return linkedContacts.build();
   }
 
   /** Loads keys to cached EPP resources by their foreign keys. */
-  private static <T extends EppResource> ImmutableMap<String, Key<T>> loadByForeignKeysCached(
+  private static <T extends EppResource> ImmutableMap<String, VKey<T>> loadByForeignKeysCached(
       final Set<String> foreignKeys, final Class<T> clazz, final DateTime now)
       throws InvalidReferencesException {
     Map<String, ForeignKeyIndex<T>> fkis = ForeignKeyIndex.loadCached(clazz, foreignKeys, now);

core/src/main/java/google/registry/model/host/HostResource.java

@@ -123,7 +123,9 @@ public class HostResource extends EppResource
     return fullyQualifiedHostName;
   }
 
-  public VKey<HostResource> createKey() {
+  @Override
+  public VKey<HostResource> createVKey() {
+    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.createOfy(HostResource.class, Key.create(this));
   }
 

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -43,6 +43,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
 import java.util.Optional;
@@ -99,7 +100,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * <p>This field holds a key to the only referenced resource. It is named "topReference" for
    * historical reasons.
    */
-  Key<E> topReference;
+  VKey<E> topReference;
 
   public String getForeignKey() {
     return foreignKey;
@@ -109,7 +110,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     return deletionTime;
   }
 
-  public Key<E> getResourceKey() {
+  public VKey<E> getResourceKey() {
     return topReference;
   }
 
@@ -125,7 +126,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     @SuppressWarnings("unchecked")
     Class<E> resourceClass = (Class<E>) resource.getClass();
     ForeignKeyIndex<E> instance = instantiate(mapToFkiClass(resourceClass));
-    instance.topReference = Key.create(resource);
+    instance.topReference = (VKey<E>) resource.createVKey();
     instance.foreignKey = resource.getForeignKey();
     instance.deletionTime = deletionTime;
     return instance;
@@ -141,18 +142,18 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   /**
    * Loads a {@link Key} to an {@link EppResource} from Datastore by foreign key.
    *
-   * <p>Returns null if no foreign key index with this foreign key was ever created, or if the
-   * most recently created foreign key index was deleted before time "now". This method does not
-   * actually check that the referenced resource actually exists. However, for normal epp resources,
-   * it is safe to assume that the referenced resource exists if the foreign key index does.
+   * <p>Returns null if no foreign key index with this foreign key was ever created, or if the most
+   * recently created foreign key index was deleted before time "now". This method does not actually
+   * check that the referenced resource actually exists. However, for normal epp resources, it is
+   * safe to assume that the referenced resource exists if the foreign key index does.
    *
    * @param clazz the resource type to load
    * @param foreignKey id to match
    * @param now the current logical time to use when checking for soft deletion of the foreign key
-   *        index
+   *     index
    */
   @Nullable
-  public static <E extends EppResource> Key<E> loadAndGetKey(
+  public static <E extends EppResource> VKey<E> loadAndGetKey(
       Class<E> clazz, String foreignKey, DateTime now) {
     ForeignKeyIndex<E> index = load(clazz, foreignKey, now);
     return (index == null) ? null : index.getResourceKey();

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -14,6 +14,7 @@
 
 package google.registry.model.ofy;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -157,7 +158,7 @@ public class DatastoreTransactionManager implements TransactionManager {
   @Override
   public <T> ImmutableList<T> load(Iterable<VKey<T>> keys) {
     Iterator<Key<T>> iter =
-        StreamSupport.stream(keys.spliterator(), false).map(key -> key.getOfyKey()).iterator();
+        StreamSupport.stream(keys.spliterator(), false).map(VKey::getOfyKey).iterator();
 
     // The lambda argument to keys() effectively converts Iterator -> Iterable.
     return ImmutableList.copyOf(getOfy().load().keys(() -> iter).values());
@@ -170,7 +171,18 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public <T> void delete(VKey<T> key) {
+  public void delete(VKey<?> key) {
     getOfy().delete().key(key.getOfyKey()).now();
   }
+
+  @Override
+  public void delete(Iterable<? extends VKey<?>> vKeys) {
+    // We have to create a list to work around the wildcard capture issue here.
+    // See https://docs.oracle.com/javase/tutorial/java/generics/capture.html
+    ImmutableList<Key<?>> list =
+        StreamSupport.stream(vKeys.spliterator(), false)
+            .map(VKey::getOfyKey)
+            .collect(toImmutableList());
+    getOfy().delete().keys(list).now();
+  }
 }

core/src/main/java/google/registry/model/poll/PendingActionNotificationResponse.java

@@ -19,6 +19,7 @@ import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
+import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
@@ -29,11 +30,12 @@ import org.joda.time.DateTime;
 
 /** The {@link ResponseData} returned when completing a pending action on a domain. */
 @XmlTransient
-public abstract class PendingActionNotificationResponse
-    extends ImmutableObject implements ResponseData {
+@Embeddable
+public class PendingActionNotificationResponse extends ImmutableObject implements ResponseData {
 
   /** The inner name type that contains a name and the result boolean. */
   @Embed
+  @Embeddable
   static class NameOrId extends ImmutableObject {
     @XmlValue
     String value;

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -25,6 +25,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
@@ -39,10 +40,23 @@ import google.registry.model.poll.PendingActionNotificationResponse.DomainPendin
 import google.registry.model.poll.PendingActionNotificationResponse.HostPendingActionNotificationResponse;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
+import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferResponse.ContactTransferResponse;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
+import google.registry.persistence.WithLongVKey;
 import java.util.List;
 import java.util.Optional;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.DiscriminatorColumn;
+import javax.persistence.DiscriminatorValue;
+import javax.persistence.Embedded;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Inheritance;
+import javax.persistence.InheritanceType;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -68,28 +82,52 @@ import org.joda.time.DateTime;
 @Entity
 @ReportedOn
 @ExternalMessagingName("message")
+@javax.persistence.Entity
+@Inheritance(strategy = InheritanceType.SINGLE_TABLE)
+@DiscriminatorColumn(name = "type")
+@javax.persistence.Table(
+    indexes = {
+      @javax.persistence.Index(columnList = "registrar_id"),
+      @javax.persistence.Index(columnList = "eventTime")
+    })
 public abstract class PollMessage extends ImmutableObject
     implements Buildable, TransferServerApproveEntity {
 
   /** Entity id. */
   @Id
-  long id;
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  @Column(name = "poll_message_id")
+  Long id;
 
-  @Parent
-  @DoNotHydrate
-  Key<HistoryEntry> parent;
+  @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
 
   /** The registrar that this poll message will be delivered to. */
   @Index
+  @Column(name = "registrar_id", nullable = false)
   String clientId;
 
   /** The time when the poll message should be delivered. May be in the future. */
   @Index
+  @Column(nullable = false)
   DateTime eventTime;
 
   /** Human readable message that will be returned with this poll message. */
+  @Column(name = "message")
   String msg;
 
+  @Ignore String domainRepoId;
+
+  @Ignore String contactRepoId;
+
+  @Ignore String hostRepoId;
+
+  @Ignore Long domainRevisionId;
+
+  @Ignore Long contactRevisionId;
+
+  @Ignore Long hostRevisionId;
+
   public Key<HistoryEntry> getParentKey() {
     return parent;
   }
@@ -180,15 +218,78 @@ public abstract class PollMessage extends ImmutableObject
    * <p>One-time poll messages are deleted from Datastore once they have been delivered and ACKed.
    */
   @EntitySubclass(index = false)
+  @javax.persistence.Entity
+  @DiscriminatorValue("ONE_TIME")
+  @WithLongVKey
   public static class OneTime extends PollMessage {
 
     // Response data. Objectify cannot persist a base class type, so we must have a separate field
     // to hold every possible derived type of ResponseData that we might store.
+    @Transient
     List<ContactPendingActionNotificationResponse> contactPendingActionNotificationResponses;
-    List<ContactTransferResponse> contactTransferResponses;
+
+    @Transient List<ContactTransferResponse> contactTransferResponses;
+
+    @Transient
     List<DomainPendingActionNotificationResponse> domainPendingActionNotificationResponses;
-    List<DomainTransferResponse> domainTransferResponses;
-    List<HostPendingActionNotificationResponse> hostPendingActionNotificationResponses;
+
+    @Transient List<DomainTransferResponse> domainTransferResponses;
+
+    @Transient List<HostPendingActionNotificationResponse> hostPendingActionNotificationResponses;
+
+    @Ignore
+    @Embedded
+    @AttributeOverrides({
+      @AttributeOverride(
+          name = "nameOrId.value",
+          column = @Column(name = "pending_action_response_name_or_id")),
+      @AttributeOverride(
+          name = "nameOrId.actionResult",
+          column = @Column(name = "pending_action_response_action_result")),
+      @AttributeOverride(
+          name = "trid.serverTransactionId",
+          column = @Column(name = "pending_action_response_server_txn_id")),
+      @AttributeOverride(
+          name = "trid.clientTransactionId",
+          column = @Column(name = "pending_action_response_client_txn_id")),
+      @AttributeOverride(
+          name = "processedDate",
+          column = @Column(name = "pending_action_response_processed_date"))
+    })
+    PendingActionNotificationResponse pendingActionNotificationResponse;
+
+    @Ignore
+    @Embedded
+    @AttributeOverrides({
+      @AttributeOverride(
+          name = "transferStatus",
+          column = @Column(name = "transfer_response_transfer_status")),
+      @AttributeOverride(
+          name = "gainingClientId",
+          column = @Column(name = "transfer_response_gaining_registrar_id")),
+      @AttributeOverride(
+          name = "transferRequestTime",
+          column = @Column(name = "transfer_response_transfer_request_time")),
+      @AttributeOverride(
+          name = "losingClientId",
+          column = @Column(name = "transfer_response_losing_registrar_id")),
+      @AttributeOverride(
+          name = "pendingTransferExpirationTime",
+          column = @Column(name = "transfer_response_pending_transfer_expiration_time"))
+    })
+    TransferResponse transferResponse;
+
+    @Ignore
+    @Column(name = "transfer_response_domain_name")
+    String fullyQualifiedDomainName;
+
+    @Ignore
+    @Column(name = "transfer_response_domain_expiration_time")
+    DateTime extendedRegistrationExpirationTime;
+
+    @Ignore
+    @Column(name = "transfer_response_contact_id")
+    String contactId;
 
     @Override
     public Builder asBuilder() {
@@ -265,9 +366,13 @@ public abstract class PollMessage extends ImmutableObject
    * happens.
    */
   @EntitySubclass(index = false)
+  @javax.persistence.Entity
+  @DiscriminatorValue("AUTORENEW")
+  @WithLongVKey
   public static class Autorenew extends PollMessage {
 
     /** The target id of the autorenew event. */
+    @Column(name = "autorenew_domain_name")
     String targetId;
 
     /** The autorenew recurs annually between {@link #eventTime} and this time. */

core/src/main/java/google/registry/model/transfer/BaseTransferObject.java

@@ -16,6 +16,8 @@ package google.registry.model.transfer;
 
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlTransient;
@@ -31,6 +33,7 @@ public abstract class BaseTransferObject extends ImmutableObject {
    * will always be non-null.
    */
   @XmlElement(name = "trStatus")
+  @Enumerated(EnumType.STRING)
   TransferStatus transferStatus;
 
   /** The gaining registrar of the current or last transfer. Can be null if never transferred. */

core/src/main/java/google/registry/model/transfer/TransferResponse.java

@@ -17,6 +17,7 @@ package google.registry.model.transfer;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.EppResource;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
+import javax.persistence.Embeddable;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
@@ -28,7 +29,8 @@ import org.joda.time.DateTime;
  * are common to all transfer responses; derived classes add resource specific fields.
  */
 @XmlTransient
-public abstract class TransferResponse extends BaseTransferObject implements ResponseData {
+@Embeddable
+public class TransferResponse extends BaseTransferObject implements ResponseData {
 
   /** An adapter to output the XML in response to a transfer command on a domain. */
   @Embed

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -278,7 +278,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
             .getResultList());
   }
 
-  private <T> int internalDelete(VKey<T> key) {
+  private int internalDelete(VKey<?> key) {
     checkArgumentNotNull(key, "key must be specified");
     assertInTransaction();
     EntityType<?> entityType = getEntityType(key.getKind());
@@ -291,10 +291,16 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public <T> void delete(VKey<T> key) {
+  public void delete(VKey<?> key) {
     internalDelete(key);
   }
 
+  @Override
+  public void delete(Iterable<? extends VKey<?>> vKeys) {
+    checkArgumentNotNull(vKeys, "vKeys must be specified");
+    vKeys.forEach(this::internalDelete);
+  }
+
   @Override
   public <T> void assertDelete(VKey<T> key) {
     if (internalDelete(key) != 1) {

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -115,7 +115,7 @@ public interface TransactionManager {
   <T> T load(VKey<T> key);
 
   /**
-   * Leads the set of entities by their key id.
+   * Loads the set of entities by their key id.
    *
    * @throws NoSuchElementException if any of the keys are not found.
    */
@@ -125,5 +125,8 @@ public interface TransactionManager {
   <T> ImmutableList<T> loadAll(Class<T> clazz);
 
   /** Deletes the entity by its id. */
-  <T> void delete(VKey<T> key);
+  void delete(VKey<?> key);
+
+  /** Deletes the set of entities by their key id. */
+  void delete(Iterable<? extends VKey<?>> keys);
 }

core/src/main/java/google/registry/rdap/RdapDomainSearchAction.java

@@ -14,6 +14,7 @@
 
 package google.registry.rdap;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -28,10 +29,10 @@ import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.InetAddresses;
 import com.google.common.primitives.Booleans;
-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.rdap.RdapJsonFormatter.OutputDataType;
 import google.registry.rdap.RdapMetrics.EndpointType;
 import google.registry.rdap.RdapMetrics.SearchType;
@@ -50,6 +51,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 import javax.inject.Inject;
 
 /**
@@ -243,7 +245,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
    */
   private DomainSearchResponse searchByNameserverLdhName(
       final RdapSearchPattern partialStringQuery) {
-    Iterable<Key<HostResource>> hostKeys = getNameserverRefsByLdhName(partialStringQuery);
+    Iterable<VKey<HostResource>> hostKeys = getNameserverRefsByLdhName(partialStringQuery);
     if (Iterables.isEmpty(hostKeys)) {
       metricInformationBuilder.setNumHostsRetrieved(0);
       throw new NotFoundException("No matching nameservers found");
@@ -261,7 +263,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
    * initial string is not required (e.g. "*.example.tld" is valid), because we can look up the
    * domain and just list all of its subordinate hosts.
    */
-  private Iterable<Key<HostResource>> getNameserverRefsByLdhName(
+  private Iterable<VKey<HostResource>> getNameserverRefsByLdhName(
       final RdapSearchPattern partialStringQuery) {
     // Handle queries without a wildcard.
     if (!partialStringQuery.getHasWildcard()) {
@@ -292,11 +294,13 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
     if (desiredRegistrar.isPresent()) {
       query = query.filter("currentSponsorClientId", desiredRegistrar.get());
     }
-    return query.keys();
+    return StreamSupport.stream(query.keys().spliterator(), false)
+        .map(key -> VKey.createOfy(HostResource.class, key))
+        .collect(toImmutableSet());
   }
 
   /** Assembles a list of {@link HostResource} keys by name when the pattern has no wildcard. */
-  private Iterable<Key<HostResource>> getNameserverRefsByLdhNameWithoutWildcard(
+  private Iterable<VKey<HostResource>> getNameserverRefsByLdhNameWithoutWildcard(
       final RdapSearchPattern partialStringQuery) {
     // If we need to check the sponsoring registrar, we need to load the resource rather than just
     // the key.
@@ -310,9 +314,9 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
       return (!host.isPresent()
               || !desiredRegistrar.get().equals(host.get().getPersistedCurrentSponsorClientId()))
           ? ImmutableList.of()
-          : ImmutableList.of(Key.create(host.get()));
+          : ImmutableList.of(host.get().createVKey());
     } else {
-      Key<HostResource> hostKey =
+      VKey<HostResource> hostKey =
           loadAndGetKey(
               HostResource.class,
               partialStringQuery.getInitialString(),
@@ -322,7 +326,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
   }
 
   /** Assembles a list of {@link HostResource} keys by name using a superordinate domain suffix. */
-  private Iterable<Key<HostResource>> getNameserverRefsByLdhNameWithSuffix(
+  private Iterable<VKey<HostResource>> getNameserverRefsByLdhNameWithSuffix(
       final RdapSearchPattern partialStringQuery) {
     // The suffix must be a domain that we manage. That way, we can look up the domain and search
     // through the subordinate hosts. This is more efficient, and lets us permit wildcard searches
@@ -338,7 +342,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
                         "A suffix in a lookup by nameserver name "
                             + "must be a domain defined in the system"));
     Optional<String> desiredRegistrar = getDesiredRegistrar();
-    ImmutableList.Builder<Key<HostResource>> builder = new ImmutableList.Builder<>();
+    ImmutableList.Builder<VKey<HostResource>> builder = new ImmutableList.Builder<>();
     for (String fqhn : ImmutableSortedSet.copyOf(domainBase.getSubordinateHosts())) {
       // We can't just check that the host name starts with the initial query string, because
       // then the query ns.exam*.example.com would match against nameserver ns.example.com.
@@ -351,10 +355,10 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
                   shouldIncludeDeleted() ? START_OF_TIME : getRequestTime());
           if (host.isPresent()
               && desiredRegistrar.get().equals(host.get().getPersistedCurrentSponsorClientId())) {
-            builder.add(Key.create(host.get()));
+            builder.add(host.get().createVKey());
           }
         } else {
-          Key<HostResource> hostKey =
+          VKey<HostResource> hostKey =
               loadAndGetKey(
                   HostResource.class,
                   fqhn,
@@ -400,7 +404,10 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
     if (desiredRegistrar.isPresent()) {
       query = query.filter("currentSponsorClientId", desiredRegistrar.get());
     }
-    return searchByNameserverRefs(query.keys());
+    return searchByNameserverRefs(
+        StreamSupport.stream(query.keys().spliterator(), false)
+            .map(key -> VKey.createOfy(HostResource.class, key))
+            .collect(toImmutableSet()));
   }
 
   /**
@@ -409,7 +416,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
    * <p>This method is called by {@link #searchByNameserverLdhName} and {@link
    * #searchByNameserverIp} after they assemble the relevant host keys.
    */
-  private DomainSearchResponse searchByNameserverRefs(final Iterable<Key<HostResource>> hostKeys) {
+  private DomainSearchResponse searchByNameserverRefs(final Iterable<VKey<HostResource>> hostKeys) {
     // We must break the query up into chunks, because the in operator is limited to 30 subqueries.
     // Since it is possible for the same domain to show up more than once in our result list (if
     // we do a wildcard nameserver search that returns multiple nameservers used by the same
@@ -420,11 +427,13 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
         ImmutableSortedSet.orderedBy(
             Comparator.comparing(DomainBase::getFullyQualifiedDomainName));
     int numHostKeysSearched = 0;
-    for (List<Key<HostResource>> chunk : Iterables.partition(hostKeys, 30)) {
+    for (List<VKey<HostResource>> chunk : Iterables.partition(hostKeys, 30)) {
       numHostKeysSearched += chunk.size();
-      Query<DomainBase> query = ofy().load()
-          .type(DomainBase.class)
-          .filter("nsHosts in", chunk);
+      Query<DomainBase> query =
+          ofy()
+              .load()
+              .type(DomainBase.class)
+              .filter("nsHosts in", chunk.stream().map(VKey::getOfyKey).collect(toImmutableSet()));
       if (!shouldIncludeDeleted()) {
         query = query.filter("deletionTime >", getRequestTime());
       // If we are not performing an inequality query, we can filter on the cursor in the query.

core/src/main/java/google/registry/tools/CommandUtilities.java

@@ -18,12 +18,12 @@ import static com.google.common.base.Preconditions.checkState;
 
 import com.google.common.base.Ascii;
 import com.google.common.base.Strings;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex;
+import google.registry.persistence.VKey;
 import org.joda.time.DateTime;
 
 /** Container class for static utility methods. */
@@ -41,7 +41,7 @@ class CommandUtilities {
       this.clazz = clazz;
     }
 
-    public Key<? extends EppResource> getKey(String uniqueId, DateTime now) {
+    public VKey<? extends EppResource> getKey(String uniqueId, DateTime now) {
       return ForeignKeyIndex.loadAndGetKey(clazz, uniqueId, now);
     }
   }

core/src/main/java/google/registry/tools/GetHistoryEntriesCommand.java

@@ -23,9 +23,9 @@ import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
 import google.registry.tools.CommandUtilities.ResourceType;
 import google.registry.xml.XmlTransformer;
 import org.joda.time.DateTime;
@@ -57,7 +57,7 @@ final class GetHistoryEntriesCommand implements CommandWithRemoteApi {
 
   @Override
   public void run() {
-    Key<? extends EppResource> parentKey = null;
+    VKey<? extends EppResource> parentKey = null;
     if (type != null || uniqueId != null) {
       checkArgument(
           type != null && uniqueId != null,
@@ -66,12 +66,12 @@ final class GetHistoryEntriesCommand implements CommandWithRemoteApi {
       checkArgumentNotNull(parentKey, "Invalid resource ID");
     }
     for (HistoryEntry entry :
-            (parentKey == null
+        (parentKey == null
                 ? ofy().load().type(HistoryEntry.class)
-                : ofy().load().type(HistoryEntry.class).ancestor(parentKey))
-        .order("modificationTime")
-        .filter("modificationTime >=", after)
-        .filter("modificationTime <=", before)) {
+                : ofy().load().type(HistoryEntry.class).ancestor(parentKey.getOfyKey()))
+            .order("modificationTime")
+            .filter("modificationTime >=", after)
+            .filter("modificationTime <=", before)) {
       System.out.printf(
           "Client: %s\nTime: %s\nClient TRID: %s\nServer TRID: %s\n%s\n",
           entry.getClientId(),

core/src/main/java/google/registry/tools/ResaveEppResourceCommand.java

@@ -14,14 +14,14 @@
 
 package google.registry.tools;
 
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
+import google.registry.persistence.VKey;
 import google.registry.tools.CommandUtilities.ResourceType;
 import org.joda.time.DateTime;
 
@@ -48,12 +48,15 @@ public final class ResaveEppResourceCommand extends MutatingCommand {
 
   @Override
   protected void init() {
-    Key<? extends EppResource> resourceKey = checkArgumentNotNull(
-        type.getKey(uniqueId, DateTime.now(UTC)),
-        "Could not find active resource of type %s: %s", type, uniqueId);
+    VKey<? extends EppResource> resourceKey =
+        checkArgumentNotNull(
+            type.getKey(uniqueId, DateTime.now(UTC)),
+            "Could not find active resource of type %s: %s",
+            type,
+            uniqueId);
     // Load the resource directly to bypass running cloneProjectedAtTime() automatically, which can
     // cause stageEntityChange() to fail due to implicit projection changes.
-    EppResource resource = ofy().load().key(resourceKey).now();
+    EppResource resource = tm().load(resourceKey);
     stageEntityChange(resource, resource);
   }
 }

core/src/main/java/google/registry/whois/DomainWhoisResponse.java

@@ -156,7 +156,7 @@ final class DomainWhoisResponse extends WhoisResponseImpl {
       // If we refer to a contact that doesn't exist, that's a bug. It means referential integrity
       // has somehow been broken. We skip the rest of this contact, but log it to hopefully bring it
       // someone's attention.
-      ContactResource contactResource = EppResource.loadCached(contact.get().getOfyKey());
+      ContactResource contactResource = EppResource.loadCached(contact.get());
       if (contactResource == null) {
         logger.atSevere().log(
             "(BUG) Broken reference found from domain %s to contact %s",

core/src/main/resources/META-INF/persistence.xml

@@ -36,6 +36,9 @@
     <class>google.registry.schema.tld.ReservedList</class>
     <class>google.registry.model.domain.secdns.DelegationSignerData</class>
     <class>google.registry.model.domain.GracePeriod</class>
+    <class>google.registry.model.poll.PollMessage</class>
+    <class>google.registry.model.poll.PollMessage$OneTime</class>
+    <class>google.registry.model.poll.PollMessage$Autorenew</class>
 
     <!-- Customized type converters -->
     <class>google.registry.persistence.converter.BillingCostTransitionConverter</class>

core/src/test/java/google/registry/batch/DeleteContactsAndHostsActionTest.java

@@ -609,7 +609,7 @@ public class DeleteContactsAndHostsActionTest
         .hasDeletionTime(END_OF_TIME);
     DomainBase domain =
         loadByForeignKey(DomainBase.class, "example.tld", clock.nowUtc()).get();
-    assertThat(domain.getNameservers()).contains(hostAfter.createKey());
+    assertThat(domain.getNameservers()).contains(hostAfter.createVKey());
     HistoryEntry historyEntry = getOnlyHistoryEntryOfType(hostAfter, HOST_DELETE_FAILURE);
     assertPollMessageFor(
         historyEntry,
@@ -679,7 +679,7 @@ public class DeleteContactsAndHostsActionTest
     persistResource(
         newDomainBase("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .setDeletionTime(clock.nowUtc().minusDays(5))
             .build());
     enqueuer.enqueueAsyncDelete(
@@ -938,7 +938,7 @@ public class DeleteContactsAndHostsActionTest
     return persistResource(
         newDomainBase(domainName, contact)
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .build());
   }
 

core/src/test/java/google/registry/beam/initsql/BackupPathsTest.java

@@ -0,0 +1,50 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.beam.initsql.BackupPaths.getKindFromFileName;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link google.registry.beam.initsql.BackupPaths}. */
+public class BackupPathsTest {
+
+  @Test
+  void getKindFromFileName_empty() {
+    assertThrows(IllegalArgumentException.class, () -> getKindFromFileName(""));
+  }
+
+  @Test
+  void getKindFromFileName_notMatch() {
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> getKindFromFileName("/tmp/all_namespaces/kind_/input-0"));
+  }
+
+  @Test
+  void getKindFromFileName_success() {
+    assertThat(getKindFromFileName("scheme:/somepath/all_namespaces/kind_mykind/input-something"))
+        .isEqualTo("mykind");
+  }
+
+  @Test
+  void getKindFromFileName_specialChar_success() {
+    assertThat(
+            getKindFromFileName("scheme:/somepath/all_namespaces/kind_.*+? /(a)/input-something"))
+        .isEqualTo(".*+? /(a)");
+  }
+}

core/src/test/java/google/registry/beam/initsql/BackupTestStore.java

@@ -0,0 +1,137 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.google.appengine.api.datastore.Entity;
+import com.googlecode.objectify.Key;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.FakeClock;
+import google.registry.tools.LevelDbFileBuilder;
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/**
+ * Wrapper of a Datastore test instance that can generate backups.
+ *
+ * <p>A Datastore backup consists of an unsynchronized data export and a sequence of incremental
+ * Commit Logs that overlap with the export process. Together they can be used to recreate a
+ * consistent snapshot of the Datastore.
+ */
+class BackupTestStore implements AutoCloseable {
+
+  private static final DateTimeFormatter EXPORT_TIMESTAMP_FORMAT =
+      DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss_SSS");
+
+  private final FakeClock fakeClock;
+  private AppEngineRule appEngine;
+
+  BackupTestStore(FakeClock fakeClock) throws Exception {
+    this.fakeClock = fakeClock;
+    this.appEngine =
+        new AppEngineRule.Builder()
+            .withDatastore()
+            .withoutCannedData()
+            .withClock(fakeClock)
+            .build();
+    this.appEngine.beforeEach(null);
+  }
+
+  /** Inserts or updates {@code entities} in the Datastore. */
+  @SafeVarargs
+  final void insertOrUpdate(Object... entities) {
+    tm().transact(() -> ofy().save().entities(entities).now());
+  }
+
+  /** Deletes {@code entities} from the Datastore. */
+  @SafeVarargs
+  final void delete(Object... entities) {
+    tm().transact(() -> ofy().delete().entities(entities).now());
+  }
+
+  /**
+   * Exports entities of the caller provided types and returns the directory where data is exported.
+   *
+   * @param exportRootPath path to the root directory of all exports. A subdirectory will be created
+   *     for this export
+   * @param pojoTypes java class of all entities to be exported
+   * @param excludes {@link Set} of {@link Key keys} of the entities not to export.This can be used
+   *     to simulate an inconsistent export
+   * @return directory where data is exported
+   */
+  File export(String exportRootPath, Iterable<Class<?>> pojoTypes, Set<Key<?>> excludes)
+      throws IOException {
+    File exportDirectory = getExportDirectory(exportRootPath);
+    for (Class<?> pojoType : pojoTypes) {
+      File perKindFile =
+          new File(
+              BackupPaths.getExportFileNameByShard(
+                  exportDirectory.getAbsolutePath(), Key.getKind(pojoType), 0));
+      checkState(
+          perKindFile.getParentFile().mkdirs(),
+          "Failed to create per-kind export directory for %s.",
+          perKindFile.getParentFile().getAbsolutePath());
+      exportOneKind(perKindFile, pojoType, excludes);
+    }
+    return exportDirectory;
+  }
+
+  private void exportOneKind(File perKindFile, Class<?> pojoType, Set<Key<?>> excludes)
+      throws IOException {
+    LevelDbFileBuilder builder = new LevelDbFileBuilder(perKindFile);
+    for (Object pojo : ofy().load().type(pojoType).iterable()) {
+      if (!excludes.contains(Key.create(pojo))) {
+        try {
+          builder.addEntity(toEntity(pojo));
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+    builder.build();
+  }
+
+  void saveCommitLog() {
+    throw new UnsupportedOperationException("Not implemented yet");
+  }
+
+  @Override
+  public void close() throws Exception {
+    if (appEngine != null) {
+      appEngine.afterEach(null);
+      appEngine = null;
+    }
+  }
+
+  private Entity toEntity(Object pojo) {
+    return tm().transactNew(() -> ofy().save().toEntity(pojo));
+  }
+
+  private File getExportDirectory(String exportRootPath) {
+    File exportDirectory =
+        new File(exportRootPath, fakeClock.nowUtc().toString(EXPORT_TIMESTAMP_FORMAT));
+    checkState(
+        exportDirectory.mkdirs(),
+        "Failed to create export directory %s.",
+        exportDirectory.getAbsolutePath());
+    return exportDirectory;
+  }
+}

core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java

@@ -0,0 +1,171 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.testing.DatastoreHelper.newContactResource;
+import static google.registry.testing.DatastoreHelper.newDomainBase;
+import static google.registry.testing.DatastoreHelper.newRegistry;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Streams;
+import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
+import com.googlecode.objectify.Key;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.registry.Registry;
+import google.registry.testing.FakeClock;
+import google.registry.tools.LevelDbLogReader;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Stream;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/** Unit tests for {@link BackupTestStore}. */
+public class BackupTestStoreTest {
+  private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z");
+
+  private FakeClock fakeClock;
+  private BackupTestStore store;
+
+  @TempDir File tempDir;
+
+  @BeforeEach
+  void beforeEach() throws Exception {
+    fakeClock = new FakeClock(START_TIME);
+    store = new BackupTestStore(fakeClock);
+
+    store.insertOrUpdate(newRegistry("tld1", "TLD1"));
+    ContactResource contact1 = newContactResource("contact_1");
+    DomainBase domain1 = newDomainBase("domain1.tld1", contact1);
+    store.insertOrUpdate(contact1, domain1);
+  }
+
+  @AfterEach
+  void afterEach() throws Exception {
+    store.close();
+  }
+
+  @Test
+  void export_filesCreated() throws IOException {
+    String exportRootPath = tempDir.getAbsolutePath();
+    File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_000");
+    assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile())
+        .that(exportFolder.exists())
+        .isFalse();
+    File actualExportFolder = export(exportRootPath, Collections.EMPTY_SET);
+    assertThat(actualExportFolder).isEquivalentAccordingToCompareTo(exportFolder);
+    try (Stream<String> files =
+        Files.walk(exportFolder.toPath())
+            .filter(Files::isRegularFile)
+            .map(Path::toString)
+            .map(string -> string.substring(exportFolder.getAbsolutePath().length()))) {
+      assertThat(files)
+          .containsExactly(
+              "/all_namespaces/kind_Registry/input-0",
+              "/all_namespaces/kind_DomainBase/input-0",
+              "/all_namespaces/kind_ContactResource/input-0");
+    }
+  }
+
+  @Test
+  void export_folderNameChangesWithTime() throws IOException {
+    String exportRootPath = tempDir.getAbsolutePath();
+    fakeClock.advanceOneMilli();
+    File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_001");
+    assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile())
+        .that(exportFolder.exists())
+        .isFalse();
+    assertThat(export(exportRootPath, Collections.EMPTY_SET))
+        .isEquivalentAccordingToCompareTo(exportFolder);
+  }
+
+  @Test
+  void export_dataReadBack() throws IOException {
+    String exportRootPath = tempDir.getAbsolutePath();
+    File exportFolder = export(exportRootPath, Collections.EMPTY_SET);
+    ImmutableList<String> tldStrings =
+        loadPropertyFromExportedEntities(
+            new File(exportFolder, "/all_namespaces/kind_Registry/input-0"),
+            Registry.class,
+            Registry::getTldStr);
+    assertThat(tldStrings).containsExactly("tld1");
+    ImmutableList<String> domainStrings =
+        loadPropertyFromExportedEntities(
+            new File(exportFolder, "/all_namespaces/kind_DomainBase/input-0"),
+            DomainBase.class,
+            DomainBase::getFullyQualifiedDomainName);
+    assertThat(domainStrings).containsExactly("domain1.tld1");
+    ImmutableList<String> contactIds =
+        loadPropertyFromExportedEntities(
+            new File(exportFolder, "/all_namespaces/kind_ContactResource/input-0"),
+            ContactResource.class,
+            ContactResource::getContactId);
+    assertThat(contactIds).containsExactly("contact_1");
+  }
+
+  @Test
+  void export_excludeSomeEntity() throws IOException {
+    store.insertOrUpdate(newRegistry("tld2", "TLD2"));
+    String exportRootPath = tempDir.getAbsolutePath();
+    File exportFolder =
+        export(
+            exportRootPath, ImmutableSet.of(Key.create(getCrossTldKey(), Registry.class, "tld1")));
+    ImmutableList<String> tlds =
+        loadPropertyFromExportedEntities(
+            new File(exportFolder, "/all_namespaces/kind_Registry/input-0"),
+            Registry.class,
+            Registry::getTldStr);
+    assertThat(tlds).containsExactly("tld2");
+  }
+
+  private File export(String exportRootPath, Set<Key<?>> excludes) throws IOException {
+    return store.export(
+        exportRootPath,
+        ImmutableList.of(ContactResource.class, DomainBase.class, Registry.class),
+        excludes);
+  }
+
+  private static <T> ImmutableList<String> loadPropertyFromExportedEntities(
+      File dataFile, Class<T> ofyEntityType, Function<T, String> getter) throws IOException {
+    return Streams.stream(LevelDbLogReader.from(dataFile.toPath()))
+        .map(bytes -> toOfyEntity(bytes, ofyEntityType))
+        .map(getter)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  private static <T> T toOfyEntity(byte[] rawRecord, Class<T> ofyEntityType) {
+    EntityProto proto = new EntityProto();
+    proto.parseFrom(rawRecord);
+    Entity entity = EntityTranslator.createFromPb(proto);
+    return ofyEntityType.cast(ofy().load().fromEntity(entity));
+  }
+}

core/src/test/java/google/registry/beam/initsql/ExportloadingTransformsTest.java

@@ -0,0 +1,201 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatastoreHelper.newContactResource;
+import static google.registry.testing.DatastoreHelper.newDomainBase;
+import static google.registry.testing.DatastoreHelper.newRegistry;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.EntityTranslator;
+import com.google.common.collect.ImmutableList;
+import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
+import com.googlecode.objectify.Key;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.registry.Registry;
+import google.registry.testing.FakeClock;
+import java.io.File;
+import java.io.Serializable;
+import java.util.Collections;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Unit tests for {@link ExportLoadingTransforms}.
+ *
+ * <p>This class implements {@link Serializable} so that test {@link DoFn} classes may be inlined.
+ */
+// TODO(weiminyu): Upgrade to JUnit5 when TestPipeline is upgraded. It is also easy to adapt with
+// a wrapper.
+@RunWith(JUnit4.class)
+public class ExportloadingTransformsTest implements Serializable {
+  private static final ImmutableList<Class<?>> ALL_KINDS =
+      ImmutableList.of(Registry.class, ContactResource.class, DomainBase.class);
+  private static final ImmutableList<String> ALL_KIND_STRS =
+      ALL_KINDS.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
+
+  @Rule public final transient TemporaryFolder exportRootDir = new TemporaryFolder();
+
+  @Rule
+  public final transient TestPipeline pipeline =
+      TestPipeline.create().enableAbandonedNodeEnforcement(true);
+
+  private FakeClock fakeClock;
+  private transient BackupTestStore store;
+  private File exportDir;
+  // Canned data that are persisted to Datastore, used by assertions in tests.
+  // TODO(weiminyu): use Ofy entity pojos directly.
+  private transient ImmutableList<Entity> persistedEntities;
+
+  @Before
+  public void beforeEach() throws Exception {
+    fakeClock = new FakeClock();
+    store = new BackupTestStore(fakeClock);
+
+    Registry registry = newRegistry("tld1", "TLD1");
+    store.insertOrUpdate(registry);
+    ContactResource contact1 = newContactResource("contact_1");
+    DomainBase domain1 = newDomainBase("domain1.tld1", contact1);
+    store.insertOrUpdate(contact1, domain1);
+    persistedEntities =
+        ImmutableList.of(registry, contact1, domain1).stream()
+            .map(ofyEntity -> tm().transact(() -> ofy().save().toEntity(ofyEntity)))
+            .collect(ImmutableList.toImmutableList());
+
+    exportDir =
+        store.export(exportRootDir.getRoot().getAbsolutePath(), ALL_KINDS, Collections.EMPTY_SET);
+  }
+
+  @After
+  public void afterEach() throws Exception {
+    if (store != null) {
+      store.close();
+      store = null;
+    }
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void getBackupDataFilePatterns() {
+    PCollection<String> patterns =
+        pipeline.apply(
+            "Get Datastore file patterns",
+            ExportLoadingTransforms.getDatastoreExportFilePatterns(
+                exportDir.getAbsolutePath(), ALL_KIND_STRS));
+
+    ImmutableList<String> expectedPatterns =
+        ImmutableList.of(
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/input-*",
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/input-*",
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_ContactResource/input-*");
+
+    PAssert.that(patterns).containsInAnyOrder(expectedPatterns);
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void getFilesByPatterns() {
+    PCollection<Metadata> fileMetas =
+        pipeline
+            .apply(
+                "File patterns to metadata",
+                Create.of(
+                        exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/input-*",
+                        exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/input-*",
+                        exportDir.getAbsolutePath()
+                            + "/all_namespaces/kind_ContactResource/input-*")
+                    .withCoder(StringUtf8Coder.of()))
+            .apply(ExportLoadingTransforms.getFilesByPatterns());
+
+    // Transform fileMetas to file names for assertions.
+    PCollection<String> fileNames =
+        fileMetas.apply(
+            "File metadata to path string",
+            ParDo.of(
+                new DoFn<Metadata, String>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element Metadata metadata, OutputReceiver<String> out) {
+                    out.output(metadata.resourceId().toString());
+                  }
+                }));
+
+    ImmutableList<String> expectedFilenames =
+        ImmutableList.of(
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/input-0",
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/input-0",
+            exportDir.getAbsolutePath() + "/all_namespaces/kind_ContactResource/input-0");
+
+    PAssert.that(fileNames).containsInAnyOrder(expectedFilenames);
+
+    pipeline.run();
+  }
+
+  @Test
+  public void loadDataFromFiles() {
+    PCollection<KV<String, byte[]>> taggedRecords =
+        pipeline
+            .apply(
+                "Get Datastore file patterns",
+                ExportLoadingTransforms.getDatastoreExportFilePatterns(
+                    exportDir.getAbsolutePath(), ALL_KIND_STRS))
+            .apply("Find Datastore files", ExportLoadingTransforms.getFilesByPatterns())
+            .apply("Load from Datastore files", ExportLoadingTransforms.loadDataFromFiles());
+
+    // Transform bytes to pojo for analysis
+    PCollection<Entity> entities =
+        taggedRecords.apply(
+            "Raw records to Entity",
+            ParDo.of(
+                new DoFn<KV<String, byte[]>, Entity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, byte[]> kv, OutputReceiver<Entity> out) {
+                    out.output(parseBytes(kv.getValue()));
+                  }
+                }));
+
+    PAssert.that(entities).containsInAnyOrder(persistedEntities);
+
+    pipeline.run();
+  }
+
+  private static Entity parseBytes(byte[] record) {
+    EntityProto proto = new EntityProto();
+    proto.parseFrom(record);
+    return EntityTranslator.createFromPb(proto);
+  }
+}

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -195,14 +195,10 @@ public class Spec11PipelineTest {
             new JSONObject()
                 .put("fullyQualifiedDomainName", "111.com")
                 .put("threatType", "MALWARE")
-                .put("threatEntryMetadata", "NONE")
-                .put("platformType", "WINDOWS")
                 .toString(),
             new JSONObject()
                 .put("fullyQualifiedDomainName", "222.com")
                 .put("threatType", "MALWARE")
-                .put("threatEntryMetadata", "NONE")
-                .put("platformType", "WINDOWS")
                 .toString());
   }
 

core/src/test/java/google/registry/dns/writer/clouddns/CloudDnsWriterTest.java

@@ -294,7 +294,7 @@ public class CloudDnsWriterTest {
 
     ImmutableSet.Builder<VKey<HostResource>> hostResourceRefBuilder = new ImmutableSet.Builder<>();
     for (HostResource nameserver : nameservers) {
-      hostResourceRefBuilder.add(nameserver.createKey());
+      hostResourceRefBuilder.add(nameserver.createVKey());
     }
 
     return newDomainBase(domainName)

core/src/test/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java

@@ -105,7 +105,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain =
         persistActiveDomain("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host1.createKey(), host2.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey(), host2.createVKey()))
             .build();
     persistResource(domain);
 
@@ -126,7 +126,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain1 =
         persistActiveDomain("example1.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host1.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey()))
             .build();
     persistResource(domain1);
 
@@ -134,7 +134,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain2 =
         persistActiveDomain("example2.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host2.createKey()))
+            .setNameservers(ImmutableSet.of(host2.createVKey()))
             .build();
     persistResource(domain2);
 
@@ -150,7 +150,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain1 =
         persistActiveDomain("example1.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host1.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey()))
             .build();
     persistResource(domain1);
 
@@ -158,7 +158,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain2 =
         persistActiveDomain("example2.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host2.createKey()))
+            .setNameservers(ImmutableSet.of(host2.createVKey()))
             .build();
     persistResource(domain2);
 
@@ -181,7 +181,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain =
         persistActiveDomain("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createVKey()))
             .setDsData(
                 ImmutableSet.of(
                     DelegationSignerData.create(1, 3, 1, base16().decode("0123456789ABCDEF"))))
@@ -206,7 +206,7 @@ public class DnsUpdateWriterTest {
         persistActiveDomain("example.tld")
             .asBuilder()
             .addStatusValue(StatusValue.SERVER_HOLD)
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createVKey()))
             .build();
     persistResource(domain);
 
@@ -250,7 +250,7 @@ public class DnsUpdateWriterTest {
         newDomainBase("example.tld")
             .asBuilder()
             .addSubordinateHost("ns1.example.tld")
-            .addNameserver(host.createKey())
+            .addNameserver(host.createVKey())
             .build());
 
     writer.publishHost("ns1.example.tld");
@@ -289,7 +289,7 @@ public class DnsUpdateWriterTest {
     persistResource(
         persistActiveDomain("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.com").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.com").createVKey()))
             .build());
 
     writer.publishHost("ns1.example.tld");
@@ -323,7 +323,8 @@ public class DnsUpdateWriterTest {
             .asBuilder()
             .addSubordinateHost("ns1.example.tld")
             .addNameservers(
-                ImmutableSet.of(externalNameserver.createKey(), inBailiwickNameserver.createKey()))
+                ImmutableSet.of(
+                    externalNameserver.createVKey(), inBailiwickNameserver.createVKey()))
             .build());
 
     writer.publishDomain("example.tld");
@@ -358,7 +359,7 @@ public class DnsUpdateWriterTest {
             .asBuilder()
             .addSubordinateHost("ns1.example.tld")
             .addSubordinateHost("foo.example.tld")
-            .addNameserver(inBailiwickNameserver.createKey())
+            .addNameserver(inBailiwickNameserver.createVKey())
             .build());
 
     writer.publishDomain("example.tld");
@@ -382,7 +383,7 @@ public class DnsUpdateWriterTest {
     DomainBase domain =
         persistActiveDomain("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createVKey()))
             .build();
     persistResource(domain);
     when(mockResolver.send(any(Message.class))).thenReturn(messageWithResponseCode(Rcode.SERVFAIL));

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -693,7 +693,7 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
         loadByForeignKey(DomainBase.class, getUniqueIdFromCommand(), clock.nowUtc())
             .get()
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .build());
     // Persist another domain that's already been deleted and references this contact and host.
     persistResource(
@@ -703,7 +703,7 @@ public class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow,
                 loadByForeignKey(ContactResource.class, "sh8013", clock.nowUtc())
                     .get()
                     .createVKey())
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .setDeletionTime(START_OF_TIME)
             .build());
     clock.advanceOneMilli();

core/src/test/java/google/registry/flows/domain/DomainInfoFlowTest.java

@@ -118,7 +118,7 @@ public class DomainInfoFlowTest extends ResourceFlowTestCase<DomainInfoFlow, Dom
                         DesignatedContact.create(Type.ADMIN, contact.createVKey()),
                         DesignatedContact.create(Type.TECH, contact.createVKey())))
                 .setNameservers(
-                    inactive ? null : ImmutableSet.of(host1.createKey(), host2.createKey()))
+                    inactive ? null : ImmutableSet.of(host1.createVKey(), host2.createVKey()))
                 .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("2fooBAR")))
                 .build());
     // Set the superordinate domain of ns1.example.com to example.com. In reality, this would have
@@ -294,7 +294,7 @@ public class DomainInfoFlowTest extends ResourceFlowTestCase<DomainInfoFlow, Dom
                 ImmutableSet.of(
                     DelegationSignerData.create(
                         12345, 3, 1, base16().decode("49FD46E6C4B45C55D4AC"))))
-            .setNameservers(ImmutableSet.of(host1.createKey(), host3.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey(), host3.createVKey()))
             .build());
     doSuccessfulTest("domain_info_response_dsdata.xml", false);
   }

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -139,7 +139,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
                         DesignatedContact.create(Type.ADMIN, mak21Contact.createVKey()),
                         DesignatedContact.create(Type.BILLING, mak21Contact.createVKey())))
                 .setRegistrant(mak21Contact.createVKey())
-                .setNameservers(ImmutableSet.of(host.createKey()))
+                .setNameservers(ImmutableSet.of(host.createVKey()))
                 .build());
     historyEntryDomainCreate =
         persistResource(
@@ -162,7 +162,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
                     ImmutableSet.of(
                         DesignatedContact.create(Type.TECH, sh8013Contact.createVKey()),
                         DesignatedContact.create(Type.ADMIN, unusedContact.createVKey())))
-                .setNameservers(ImmutableSet.of(host.createKey()))
+                .setNameservers(ImmutableSet.of(host.createVKey()))
                 .build());
     historyEntryDomainCreate =
         persistResource(
@@ -263,7 +263,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
             loadByForeignKey(
                     HostResource.class, String.format("ns%d.example.foo", i), clock.nowUtc())
                 .get()
-                .createKey());
+                .createVKey());
       }
     }
     persistResource(
@@ -290,7 +290,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
     for (int i = 0; i < 26; i++) {
       HostResource host = persistActiveHost(String.format("max_test_%d.example.tld", i));
       if (i < 13) {
-        nameservers.add(host.createKey());
+        nameservers.add(host.createVKey());
       }
     }
     ImmutableList.Builder<DesignatedContact> contactsBuilder = new ImmutableList.Builder<>();
@@ -378,13 +378,13 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
                 ImmutableSet.of(
                     loadByForeignKey(HostResource.class, "ns1.example.tld", clock.nowUtc())
                         .get()
-                        .createKey()))
+                        .createVKey()))
             .build());
     clock.advanceOneMilli();
     assertTransactionalFlow(true);
     runFlowAssertResponse(loadFile("generic_success_response.xml"));
     domain = reloadResourceByForeignKey();
-    assertThat(domain.getNameservers()).containsExactly(addedHost.createKey());
+    assertThat(domain.getNameservers()).containsExactly(addedHost.createVKey());
     assertThat(domain.getSubordinateHosts()).containsExactly("ns1.example.tld", "ns2.example.tld");
     HostResource existingHost =
         loadByForeignKey(HostResource.class, "ns1.example.tld", clock.nowUtc()).get();
@@ -1061,7 +1061,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
                 ImmutableSet.of(
                     loadByForeignKey(HostResource.class, "ns1.example.foo", clock.nowUtc())
                         .get()
-                        .createKey()))
+                        .createVKey()))
             .build());
     EppException thrown = assertThrows(AddRemoveSameValueException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
@@ -1202,13 +1202,13 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
         .doesNotContain(
             loadByForeignKey(HostResource.class, "ns2.example.foo", clock.nowUtc())
                 .get()
-                .createKey());
+                .createVKey());
     runFlow();
     assertThat(reloadResourceByForeignKey().getNameservers())
         .contains(
             loadByForeignKey(HostResource.class, "ns2.example.foo", clock.nowUtc())
                 .get()
-                .createKey());
+                .createVKey());
   }
 
   @Test
@@ -1279,7 +1279,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
             .addNameserver(
                 loadByForeignKey(HostResource.class, "ns2.example.foo", clock.nowUtc())
                     .get()
-                    .createKey())
+                    .createVKey())
             .build());
     persistResource(
         Registry.get("tld")
@@ -1291,7 +1291,7 @@ public class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow,
         .contains(
             loadByForeignKey(HostResource.class, "ns1.example.foo", clock.nowUtc())
                 .get()
-                .createKey());
+                .createVKey());
     clock.advanceOneMilli();
     runFlow();
     assertThat(reloadResourceByForeignKey().getNameservers())

core/src/test/java/google/registry/flows/host/HostDeleteFlowTest.java

@@ -284,7 +284,7 @@ public class HostDeleteFlowTest extends ResourceFlowTestCase<HostDeleteFlow, Hos
     persistResource(
         newDomainBase("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createKey()))
+            .setNameservers(ImmutableSet.of(persistActiveHost("ns1.example.tld").createVKey()))
             .build());
     EppException thrown = assertThrows(ResourceToDeleteIsReferencedException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();

core/src/test/java/google/registry/flows/host/HostInfoFlowTest.java

@@ -92,7 +92,7 @@ public class HostInfoFlowTest extends ResourceFlowTestCase<HostInfoFlow, HostRes
     persistResource(
         newDomainBase("example.foobar")
             .asBuilder()
-            .addNameserver(persistHostResource().createKey())
+            .addNameserver(persistHostResource().createVKey())
             .build());
     assertTransactionalFlow(false);
     // Check that the persisted host info was returned.

core/src/test/java/google/registry/flows/host/HostUpdateFlowTest.java

@@ -178,7 +178,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
     // The old ForeignKeyIndex is invalidated at the time we did the rename.
     ForeignKeyIndex<HostResource> oldFkiBeforeRename =
         ForeignKeyIndex.load(HostResource.class, oldHostName(), clock.nowUtc().minusMillis(1));
-    assertThat(oldFkiBeforeRename.getResourceKey()).isEqualTo(Key.create(renamedHost));
+    assertThat(oldFkiBeforeRename.getResourceKey()).isEqualTo(renamedHost.createVKey());
     assertThat(oldFkiBeforeRename.getDeletionTime()).isEqualTo(clock.nowUtc());
     ForeignKeyIndex<HostResource> oldFkiAfterRename =
         ForeignKeyIndex.load(HostResource.class, oldHostName(), clock.nowUtc());
@@ -195,7 +195,7 @@ public class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, Hos
         newDomainBase("test.xn--q9jyb4c")
             .asBuilder()
             .setDeletionTime(END_OF_TIME)
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .build());
     HostResource renamedHost = doSuccessfulTest();
     assertThat(renamedHost.isSubordinate()).isTrue();

core/src/test/java/google/registry/model/EppResourceTest.java

@@ -22,7 +22,6 @@ import static google.registry.testing.DatastoreHelper.persistActiveHost;
 import static google.registry.testing.DatastoreHelper.persistResource;
 
 import com.google.common.collect.ImmutableList;
-import com.googlecode.objectify.Key;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.host.HostResource;
 import google.registry.testing.TestCacheRule;
@@ -40,12 +39,12 @@ public class EppResourceTest extends EntityTestCase {
   @Test
   public void test_loadCached_ignoresContactChange() {
     ContactResource originalContact = persistActiveContact("contact123");
-    assertThat(EppResource.loadCached(ImmutableList.of(Key.create(originalContact))))
-        .containsExactly(Key.create(originalContact), originalContact);
+    assertThat(EppResource.loadCached(ImmutableList.of(originalContact.createVKey())))
+        .containsExactly(originalContact.createVKey(), originalContact);
     ContactResource modifiedContact =
         persistResource(originalContact.asBuilder().setEmailAddress("different@fake.lol").build());
-    assertThat(EppResource.loadCached(ImmutableList.of(Key.create(originalContact))))
-        .containsExactly(Key.create(originalContact), originalContact);
+    assertThat(EppResource.loadCached(ImmutableList.of(originalContact.createVKey())))
+        .containsExactly(originalContact.createVKey(), originalContact);
     assertThat(loadByForeignKey(ContactResource.class, "contact123", fakeClock.nowUtc()))
         .hasValue(modifiedContact);
   }
@@ -53,13 +52,13 @@ public class EppResourceTest extends EntityTestCase {
   @Test
   public void test_loadCached_ignoresHostChange() {
     HostResource originalHost = persistActiveHost("ns1.example.com");
-    assertThat(EppResource.loadCached(ImmutableList.of(Key.create(originalHost))))
-        .containsExactly(Key.create(originalHost), originalHost);
+    assertThat(EppResource.loadCached(ImmutableList.of(originalHost.createVKey())))
+        .containsExactly(originalHost.createVKey(), originalHost);
     HostResource modifiedHost =
         persistResource(
             originalHost.asBuilder().setLastTransferTime(fakeClock.nowUtc().minusDays(60)).build());
-    assertThat(EppResource.loadCached(ImmutableList.of(Key.create(originalHost))))
-        .containsExactly(Key.create(originalHost), originalHost);
+    assertThat(EppResource.loadCached(ImmutableList.of(originalHost.createVKey())))
+        .containsExactly(originalHost.createVKey(), originalHost);
     assertThat(loadByForeignKey(HostResource.class, "ns1.example.com", fakeClock.nowUtc()))
         .hasValue(modifiedHost);
   }

core/src/test/java/google/registry/model/domain/DomainBaseTest.java

@@ -79,7 +79,7 @@ public class DomainBaseTest extends EntityTestCase {
                     .setSuperordinateDomain(domainKey)
                     .setRepoId("1-COM")
                     .build())
-            .createKey();
+            .createVKey();
     VKey<ContactResource> contact1Key =
         persistResource(
                 new ContactResource.Builder()
@@ -221,7 +221,7 @@ public class DomainBaseTest extends EntityTestCase {
     assertThat(
             newDomainBase("example.com")
                 .asBuilder()
-                .setNameservers(ImmutableSet.of(newHostResource("foo.example.tld").createKey()))
+                .setNameservers(ImmutableSet.of(newHostResource("foo.example.tld").createVKey()))
                 .build()
                 .nsHosts)
         .isNotNull();
@@ -269,7 +269,7 @@ public class DomainBaseTest extends EntityTestCase {
   @Test
   public void testImplicitStatusValues() {
     ImmutableSet<VKey<HostResource>> nameservers =
-        ImmutableSet.of(newHostResource("foo.example.tld").createKey());
+        ImmutableSet.of(newHostResource("foo.example.tld").createVKey());
     StatusValue[] statuses = {StatusValue.OK};
     // OK is implicit if there's no other statuses but there are nameservers.
     assertAboutDomains()

core/src/test/java/google/registry/model/index/ForeignKeyIndexTest.java

@@ -17,7 +17,7 @@ package google.registry.model.index;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatastoreHelper.createTld;
 import static google.registry.testing.DatastoreHelper.deleteResource;
 import static google.registry.testing.DatastoreHelper.persistActiveContact;
@@ -27,7 +27,6 @@ import static google.registry.testing.DatastoreHelper.persistResource;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.common.collect.ImmutableList;
-import com.googlecode.objectify.Key;
 import google.registry.model.EntityTestCase;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.host.HostResource;
@@ -56,7 +55,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
     HostResource host = persistActiveHost("ns1.example.com");
     ForeignKeyIndex<HostResource> fki =
         ForeignKeyIndex.load(HostResource.class, "ns1.example.com", fakeClock.nowUtc());
-    assertThat(ofy().load().key(fki.getResourceKey()).now()).isEqualTo(host);
+    assertThat(tm().load(fki.getResourceKey())).isEqualTo(host);
     assertThat(fki.getDeletionTime()).isEqualTo(END_OF_TIME);
   }
 
@@ -89,7 +88,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
     fakeClock.advanceOneMilli();
     ForeignKeyHostIndex fki = new ForeignKeyHostIndex();
     fki.foreignKey = "ns1.example.com";
-    fki.topReference = Key.create(host1);
+    fki.topReference = host1.createVKey();
     fki.deletionTime = fakeClock.nowUtc();
     persistResource(fki);
     assertThat(ForeignKeyIndex.load(HostResource.class, "ns1.example.com", fakeClock.nowUtc()))

core/src/test/java/google/registry/model/poll/PollMessageTest.java

@@ -16,24 +16,31 @@ package google.registry.model.poll;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.testing.DatastoreHelper.createTld;
 import static google.registry.testing.DatastoreHelper.persistActiveDomain;
 import static google.registry.testing.DatastoreHelper.persistResource;
+import static google.registry.testing.SqlHelper.saveRegistrar;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import google.registry.model.EntityTestCase;
 import google.registry.model.domain.Period;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.reporting.HistoryEntry;
-import org.junit.Before;
-import org.junit.Test;
+import google.registry.persistence.VKey;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link PollMessage}. */
 public class PollMessageTest extends EntityTestCase {
 
   HistoryEntry historyEntry;
 
-  @Before
+  public PollMessageTest() {
+    super(true);
+  }
+
+  @BeforeEach
   public void setUp() {
     createTld("foobar");
     historyEntry =
@@ -52,6 +59,50 @@ public class PollMessageTest extends EntityTestCase {
                 .build());
   }
 
+  @Test
+  void testCloudSqlPersistenceOneTime() {
+    saveRegistrar("TheRegistrar");
+    PollMessage.OneTime pollMessage =
+        new PollMessage.OneTime.Builder()
+            .setClientId("TheRegistrar")
+            .setEventTime(fakeClock.nowUtc())
+            .setMsg("Test poll message")
+            .setParent(historyEntry)
+            .build();
+    pollMessage.id = null;
+    jpaTm().transact(() -> jpaTm().saveNew(pollMessage));
+    PollMessage.OneTime persisted =
+        jpaTm()
+            .transact(
+                () -> jpaTm().load(VKey.createSql(PollMessage.OneTime.class, pollMessage.id)));
+    persisted.id = pollMessage.id;
+    persisted.parent = pollMessage.parent;
+    assertThat(persisted).isEqualTo(pollMessage);
+  }
+
+  @Test
+  void testCloudSqlPersistenceAutorenew() {
+    saveRegistrar("TheRegistrar");
+    PollMessage.Autorenew pollMessage =
+        new PollMessage.Autorenew.Builder()
+            .setClientId("TheRegistrar")
+            .setEventTime(fakeClock.nowUtc())
+            .setMsg("Test poll message")
+            .setParent(historyEntry)
+            .setAutorenewEndTime(fakeClock.nowUtc().plusDays(365))
+            .setTargetId("foobar.foo")
+            .build();
+    pollMessage.id = null;
+    jpaTm().transact(() -> jpaTm().saveNew(pollMessage));
+    PollMessage.Autorenew persisted =
+        jpaTm()
+            .transact(
+                () -> jpaTm().load(VKey.createSql(PollMessage.Autorenew.class, pollMessage.id)));
+    persisted.id = pollMessage.id;
+    persisted.parent = pollMessage.parent;
+    assertThat(persisted).isEqualTo(pollMessage);
+  }
+
   @Test
   public void testPersistenceOneTime() {
     PollMessage.OneTime pollMessage =

core/src/test/java/google/registry/persistence/transaction/JpaEntityCoverage.java

@@ -28,6 +28,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Supplier;
+import javax.persistence.DiscriminatorValue;
 import javax.persistence.Entity;
 import org.junit.rules.ExternalResource;
 
@@ -46,6 +47,7 @@ public class JpaEntityCoverage extends ExternalResource {
       PersistenceXmlUtility.getManagedClasses().stream()
           .filter(e -> !IGNORE_ENTITIES.contains(e.getSimpleName()))
           .filter(e -> e.isAnnotationPresent(Entity.class))
+          .filter(e -> !e.isAnnotationPresent(DiscriminatorValue.class))
           .collect(ImmutableSet.toImmutableSet());
   private static final Set<Class> allCoveredJpaEntities = Sets.newHashSet();
   // Map of test class name to boolean flag indicating if it tests any JPA entities.

core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java

@@ -14,6 +14,8 @@
 
 package google.registry.persistence.transaction;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.junit.Assert.assertThrows;
@@ -30,7 +32,9 @@ import google.registry.testing.AppEngineRule;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectRule;
+import java.util.List;
 import java.util.NoSuchElementException;
+import java.util.Set;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.TestTemplate;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -62,39 +66,38 @@ public class TransactionManagerTest {
           .withJpaUnitTestEntities(TestEntity.class)
           .build();
 
-  public TransactionManagerTest() {}
+  TransactionManagerTest() {}
 
   @BeforeEach
-  public void setUp() {
+  void setUp() {
     inject.setStaticField(Ofy.class, "clock", fakeClock);
+    fakeClock.advanceOneMilli();
   }
 
   @TestTemplate
-  public void inTransaction_returnsCorrespondingResult() {
+  void inTransaction_returnsCorrespondingResult() {
     assertThat(tm().inTransaction()).isFalse();
     tm().transact(() -> assertThat(tm().inTransaction()).isTrue());
     assertThat(tm().inTransaction()).isFalse();
   }
 
   @TestTemplate
-  public void assertInTransaction_throwsExceptionWhenNotInTransaction() {
+  void assertInTransaction_throwsExceptionWhenNotInTransaction() {
     assertThrows(IllegalStateException.class, () -> tm().assertInTransaction());
     tm().transact(() -> tm().assertInTransaction());
     assertThrows(IllegalStateException.class, () -> tm().assertInTransaction());
   }
 
   @TestTemplate
-  public void getTransactionTime_throwsExceptionWhenNotInTransaction() {
-    FakeClock txnClock = fakeClock;
-    txnClock.advanceOneMilli();
+  void getTransactionTime_throwsExceptionWhenNotInTransaction() {
     assertThrows(IllegalStateException.class, () -> tm().getTransactionTime());
-    tm().transact(() -> assertThat(tm().getTransactionTime()).isEqualTo(txnClock.nowUtc()));
+    tm().transact(() -> assertThat(tm().getTransactionTime()).isEqualTo(fakeClock.nowUtc()));
     assertThrows(IllegalStateException.class, () -> tm().getTransactionTime());
   }
 
   @TestTemplate
-  public void transact_hasNoEffectWithPartialSuccess() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
+  void transact_hasNoEffectWithPartialSuccess() {
+    assertEntityNotExist(theEntity);
     assertThrows(
         RuntimeException.class,
         () ->
@@ -104,152 +107,162 @@ public class TransactionManagerTest {
                       tm().saveNew(theEntity);
                       throw new RuntimeException();
                     }));
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
+    assertEntityNotExist(theEntity);
   }
 
   @TestTemplate
-  public void transact_reusesExistingTransaction() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
-    fakeClock.advanceOneMilli();
+  void transact_reusesExistingTransaction() {
+    assertEntityNotExist(theEntity);
     tm().transact(() -> tm().transact(() -> tm().saveNew(theEntity)));
-    fakeClock.advanceOneMilli();
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isTrue();
+    assertEntityExists(theEntity);
   }
 
   @TestTemplate
-  public void saveNew_succeeds() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
-    fakeClock.advanceOneMilli();
+  void saveNew_succeeds() {
+    assertEntityNotExist(theEntity);
     tm().transact(() -> tm().saveNew(theEntity));
-    fakeClock.advanceOneMilli();
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isTrue();
-    fakeClock.advanceOneMilli();
+    assertEntityExists(theEntity);
     assertThat(tm().transact(() -> tm().load(theEntity.key()))).isEqualTo(theEntity);
   }
 
   @TestTemplate
-  public void saveAllNew_succeeds() {
-    moreEntities.forEach(
-        entity -> assertThat(tm().transact(() -> tm().checkExists(entity))).isFalse());
-    fakeClock.advanceOneMilli();
+  void saveAllNew_succeeds() {
+    assertAllEntitiesNotExist(moreEntities);
     tm().transact(() -> tm().saveAllNew(moreEntities));
-    fakeClock.advanceOneMilli();
-    moreEntities.forEach(
-        entity -> assertThat(tm().transact(() -> tm().checkExists(entity))).isTrue());
+    assertAllEntitiesExist(moreEntities);
   }
 
   @TestTemplate
-  public void saveNewOrUpdate_persistsNewEntity() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
-    fakeClock.advanceOneMilli();
+  void saveNewOrUpdate_persistsNewEntity() {
+    assertEntityNotExist(theEntity);
     tm().transact(() -> tm().saveNewOrUpdate(theEntity));
-    fakeClock.advanceOneMilli();
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isTrue();
-    fakeClock.advanceOneMilli();
+    assertEntityExists(theEntity);
     assertThat(tm().transact(() -> tm().load(theEntity.key()))).isEqualTo(theEntity);
   }
 
   @TestTemplate
-  public void saveNewOrUpdate_updatesExistingEntity() {
-    fakeClock.advanceOneMilli();
+  void saveNewOrUpdate_updatesExistingEntity() {
     tm().transact(() -> tm().saveNew(theEntity));
-    fakeClock.advanceOneMilli();
     TestEntity persisted = tm().transact(() -> tm().load(theEntity.key()));
     assertThat(persisted.data).isEqualTo("foo");
     theEntity.data = "bar";
+    fakeClock.advanceOneMilli();
     tm().transact(() -> tm().saveNewOrUpdate(theEntity));
-    fakeClock.advanceOneMilli();
     persisted = tm().transact(() -> tm().load(theEntity.key()));
-    fakeClock.advanceOneMilli();
     assertThat(persisted.data).isEqualTo("bar");
   }
 
   @TestTemplate
-  public void saveNewOrUpdateAll_succeeds() {
-    moreEntities.forEach(
-        entity -> assertThat(tm().transact(() -> tm().checkExists(entity))).isFalse());
-    fakeClock.advanceOneMilli();
+  void saveNewOrUpdateAll_succeeds() {
+    assertAllEntitiesNotExist(moreEntities);
     tm().transact(() -> tm().saveNewOrUpdateAll(moreEntities));
-    fakeClock.advanceOneMilli();
-    moreEntities.forEach(
-        entity -> assertThat(tm().transact(() -> tm().checkExists(entity))).isTrue());
+    assertAllEntitiesExist(moreEntities);
   }
 
   @TestTemplate
-  public void update_succeeds() {
-    fakeClock.advanceOneMilli();
+  void update_succeeds() {
     tm().transact(() -> tm().saveNew(theEntity));
-    fakeClock.advanceOneMilli();
     TestEntity persisted =
         tm().transact(
                 () ->
                     tm().load(
                             VKey.create(TestEntity.class, theEntity.name, Key.create(theEntity))));
-    fakeClock.advanceOneMilli();
     assertThat(persisted.data).isEqualTo("foo");
     theEntity.data = "bar";
-    tm().transact(() -> tm().update(theEntity));
     fakeClock.advanceOneMilli();
+    tm().transact(() -> tm().update(theEntity));
     persisted = tm().transact(() -> tm().load(theEntity.key()));
     assertThat(persisted.data).isEqualTo("bar");
   }
 
   @TestTemplate
-  public void load_succeeds() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
-    fakeClock.advanceOneMilli();
+  void load_succeeds() {
+    assertEntityNotExist(theEntity);
     tm().transact(() -> tm().saveNew(theEntity));
-    fakeClock.advanceOneMilli();
     TestEntity persisted = tm().transact(() -> tm().load(theEntity.key()));
     assertThat(persisted.name).isEqualTo("theEntity");
     assertThat(persisted.data).isEqualTo("foo");
   }
 
   @TestTemplate
-  public void load_throwsOnMissingElement() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
-    fakeClock.advanceOneMilli();
+  void load_throwsOnMissingElement() {
+    assertEntityNotExist(theEntity);
     assertThrows(
         NoSuchElementException.class, () -> tm().transact(() -> tm().load(theEntity.key())));
   }
 
   @TestTemplate
-  public void maybeLoad_succeeds() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
-    fakeClock.advanceOneMilli();
+  void maybeLoad_succeeds() {
+    assertEntityNotExist(theEntity);
     tm().transact(() -> tm().saveNew(theEntity));
-    fakeClock.advanceOneMilli();
     TestEntity persisted = tm().transact(() -> tm().maybeLoad(theEntity.key()).get());
     assertThat(persisted.name).isEqualTo("theEntity");
     assertThat(persisted.data).isEqualTo("foo");
   }
 
   @TestTemplate
-  public void maybeLoad_nonExistentObject() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
-    fakeClock.advanceOneMilli();
+  void maybeLoad_nonExistentObject() {
+    assertEntityNotExist(theEntity);
     assertThat(tm().transact(() -> tm().maybeLoad(theEntity.key())).isPresent()).isFalse();
   }
 
   @TestTemplate
-  public void delete_succeeds() {
-    fakeClock.advanceOneMilli();
+  void delete_succeeds() {
     tm().transact(() -> tm().saveNew(theEntity));
-    fakeClock.advanceOneMilli();
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isTrue();
+    assertEntityExists(theEntity);
     fakeClock.advanceOneMilli();
     tm().transact(() -> tm().delete(theEntity.key()));
-    fakeClock.advanceOneMilli();
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
+    assertEntityNotExist(theEntity);
   }
 
   @TestTemplate
-  public void delete_returnsZeroWhenNoEntity() {
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
-    fakeClock.advanceOneMilli();
+  void delete_doNothingWhenEntityNotExist() {
+    assertEntityNotExist(theEntity);
     tm().transact(() -> tm().delete(theEntity.key()));
+    assertEntityNotExist(theEntity);
+  }
+
+  @TestTemplate
+  void delete_succeedsForEntitySet() {
+    assertAllEntitiesNotExist(moreEntities);
+    tm().transact(() -> tm().saveAllNew(moreEntities));
+    Set<VKey<TestEntity>> keys =
+        moreEntities.stream().map(TestEntity::key).collect(toImmutableSet());
+    assertAllEntitiesExist(moreEntities);
     fakeClock.advanceOneMilli();
-    assertThat(tm().transact(() -> tm().checkExists(theEntity))).isFalse();
+    tm().transact(() -> tm().delete(keys));
+    assertAllEntitiesNotExist(moreEntities);
+  }
+
+  @TestTemplate
+  void delete_ignoreNonExistentEntity() {
+    assertAllEntitiesNotExist(moreEntities);
+    tm().transact(() -> tm().saveAllNew(moreEntities));
+    List<VKey<TestEntity>> keys =
+        moreEntities.stream().map(TestEntity::key).collect(toImmutableList());
+    assertAllEntitiesExist(moreEntities);
+    fakeClock.advanceOneMilli();
+    tm().transact(() -> tm().delete(keys.get(0)));
+    assertEntityNotExist(moreEntities.get(0));
+    fakeClock.advanceOneMilli();
+    tm().transact(() -> tm().delete(keys));
+    assertAllEntitiesNotExist(moreEntities);
+  }
+
+  private static void assertEntityExists(TestEntity entity) {
+    assertThat(tm().transact(() -> tm().checkExists(entity))).isTrue();
+  }
+
+  private static void assertEntityNotExist(TestEntity entity) {
+    assertThat(tm().transact(() -> tm().checkExists(entity))).isFalse();
+  }
+
+  private static void assertAllEntitiesExist(ImmutableList<TestEntity> entities) {
+    entities.forEach(TransactionManagerTest::assertEntityExists);
+  }
+
+  private static void assertAllEntitiesNotExist(ImmutableList<TestEntity> entities) {
+    entities.forEach(TransactionManagerTest::assertEntityNotExist);
   }
 
   @Entity(name = "TestEntity")

core/src/test/java/google/registry/rdap/RdapDomainSearchActionTest.java

@@ -418,7 +418,7 @@ public class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDom
       subordinateHostnamesBuilder.add(hostName);
       HostResource host = makeAndPersistHostResource(
           hostName, String.format("5.5.%d.%d", 5 + i / 250, i % 250), clock.nowUtc().minusYears(1));
-      hostKeysBuilder.add(host.createKey());
+      hostKeysBuilder.add(host.createVKey());
     }
     ImmutableSet<VKey<HostResource>> hostKeys = hostKeysBuilder.build();
     // Create all the domains at once, then persist them in parallel, for increased efficiency.

core/src/test/java/google/registry/rde/DomainBaseToXjcConverterTest.java

@@ -269,9 +269,9 @@ public class DomainBaseToXjcConverterTest {
             .setNameservers(
                 ImmutableSet.of(
                     makeHostResource(clock, "3-Q9JYB4C", "bird.or.devil.みんな", "1.2.3.4")
-                        .createKey(),
+                        .createVKey(),
                     makeHostResource(clock, "4-Q9JYB4C", "ns2.cat.みんな", "bad:f00d:cafe::15:beef")
-                        .createKey()))
+                        .createVKey()))
             .setRegistrant(
                 makeContactResource(
                         clock, "12-Q9JYB4C", "5372808-ERL", "(◕‿◕) nevermore", "prophet@evil.みんな")

core/src/test/java/google/registry/rde/RdeFixtures.java

@@ -116,8 +116,8 @@ final class RdeFixtures {
             .setIdnTableName("extended_latin")
             .setNameservers(
                 ImmutableSet.of(
-                    makeHostResource(clock, "bird.or.devil.みんな", "1.2.3.4").createKey(),
-                    makeHostResource(clock, "ns2.cat.みんな", "bad:f00d:cafe::15:beef").createKey()))
+                    makeHostResource(clock, "bird.or.devil.みんな", "1.2.3.4").createVKey(),
+                    makeHostResource(clock, "ns2.cat.みんな", "bad:f00d:cafe::15:beef").createVKey()))
             .setRegistrationExpirationTime(DateTime.parse("1994-01-01T00:00:00Z"))
             .setGracePeriods(
                 ImmutableSet.of(

core/src/test/java/google/registry/reporting/spec11/Spec11EmailUtilsTest.java

@@ -376,7 +376,7 @@ public class Spec11EmailUtilsTest {
     return persistResource(
         newDomainBase(domainName)
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host.createKey()))
+            .setNameservers(ImmutableSet.of(host.createVKey()))
             .build());
   }
 }

core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java

@@ -77,6 +77,26 @@ public class Spec11RegistrarThreatMatchesParserTest {
         .hasValue(LocalDate.parse("2018-07-14"));
   }
 
+  @Test
+  public void testSuccess_ignoreExtraFields() throws Exception {
+    ThreatMatch objectWithExtraFields =
+        ThreatMatch.fromJSON(
+            new JSONObject(
+                ImmutableMap.of(
+                    "threatType", "MALWARE",
+                    "platformType", "ANY_PLATFORM",
+                    "threatEntryMetaData", "NONE",
+                    "fullyQualifiedDomainName", "c.com")));
+    ThreatMatch objectWithoutExtraFields =
+        ThreatMatch.fromJSON(
+            new JSONObject(
+                ImmutableMap.of(
+                    "threatType", "MALWARE",
+                    "fullyQualifiedDomainName", "c.com")));
+
+    assertThat(objectWithExtraFields).isEqualTo(objectWithoutExtraFields);
+  }
+
   /** The expected contents of the sample spec11 report file */
   static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
     return ImmutableSet.of(getMatchA(), getMatchB());
@@ -90,8 +110,6 @@ public class Spec11RegistrarThreatMatchesParserTest {
                 new JSONObject(
                     ImmutableMap.of(
                         "threatType", "MALWARE",
-                        "platformType", "ANY_PLATFORM",
-                        "threatEntryMetadata", "NONE",
                         "fullyQualifiedDomainName", "a.com")))));
   }
 
@@ -103,15 +121,11 @@ public class Spec11RegistrarThreatMatchesParserTest {
                 new JSONObject(
                     ImmutableMap.of(
                         "threatType", "MALWARE",
-                        "platformType", "ANY_PLATFORM",
-                        "threatEntryMetadata", "NONE",
                         "fullyQualifiedDomainName", "b.com"))),
             ThreatMatch.fromJSON(
                 new JSONObject(
                     ImmutableMap.of(
                         "threatType", "MALWARE",
-                        "platformType", "ANY_PLATFORM",
-                        "threatEntryMetadata", "NONE",
                         "fullyQualifiedDomainName", "c.com")))));
   }
 

core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java

@@ -19,6 +19,7 @@ import static com.google.common.truth.Truth.assert_;
 import google.registry.model.billing.BillingEventTest;
 import google.registry.model.contact.ContactResourceTest;
 import google.registry.model.domain.DomainBaseSqlTest;
+import google.registry.model.poll.PollMessageTest;
 import google.registry.model.registry.RegistryLockDaoTest;
 import google.registry.persistence.transaction.JpaEntityCoverage;
 import google.registry.schema.cursor.CursorDaoTest;
@@ -75,6 +76,7 @@ import org.junit.runner.RunWith;
   CursorDaoTest.class,
   DomainBaseSqlTest.class,
   LockDaoTest.class,
+  PollMessageTest.class,
   PremiumListDaoTest.class,
   RegistrarDaoTest.class,
   RegistryLockDaoTest.class,

core/src/test/java/google/registry/server/Fixture.java

@@ -131,8 +131,8 @@ public enum Fixture {
                       DesignatedContact.create(TECH, justine.createVKey())))
               .setNameservers(
                   ImmutableSet.of(
-                      persistActiveHost("ns1.love.xn--q9jyb4c").createKey(),
-                      persistActiveHost("ns2.love.xn--q9jyb4c").createKey()))
+                      persistActiveHost("ns1.love.xn--q9jyb4c").createVKey(),
+                      persistActiveHost("ns2.love.xn--q9jyb4c").createVKey()))
               .build());
 
       persistResource(
@@ -145,11 +145,11 @@ public enum Fixture {
                       DesignatedContact.create(TECH, justine.createVKey())))
               .setNameservers(
                   ImmutableSet.of(
-                      persistActiveHost("ns1.linode.com").createKey(),
-                      persistActiveHost("ns2.linode.com").createKey(),
-                      persistActiveHost("ns3.linode.com").createKey(),
-                      persistActiveHost("ns4.linode.com").createKey(),
-                      persistActiveHost("ns5.linode.com").createKey()))
+                      persistActiveHost("ns1.linode.com").createVKey(),
+                      persistActiveHost("ns2.linode.com").createVKey(),
+                      persistActiveHost("ns3.linode.com").createVKey(),
+                      persistActiveHost("ns4.linode.com").createVKey(),
+                      persistActiveHost("ns5.linode.com").createVKey()))
               .build());
 
       persistResource(

core/src/test/java/google/registry/testing/AppEngineRule.java

@@ -123,7 +123,9 @@ public final class AppEngineRule extends ExternalResource
 
   JpaUnitTestRule jpaUnitTestRule;
 
-  private boolean withDatastoreAndCloudSql;
+  private boolean withDatastore;
+  private boolean withoutCannedData;
+  private boolean withCloudSql;
   private boolean enableJpaEntityCoverageCheck;
   private boolean withJpaUnitTest;
   private boolean withLocalModules;
@@ -148,9 +150,22 @@ public final class AppEngineRule extends ExternalResource
 
     /** Turn on the Datastore service and the Cloud SQL service. */
     public Builder withDatastoreAndCloudSql() {
-      rule.withDatastoreAndCloudSql = true;
+      rule.withDatastore = rule.withCloudSql = true;
       return this;
     }
+
+    /** Turns on Datastore only, for use by test data generators. */
+    public Builder withDatastore() {
+      rule.withDatastore = true;
+      return this;
+    }
+
+    /** Disables insertion of canned data. */
+    public Builder withoutCannedData() {
+      rule.withoutCannedData = true;
+      return this;
+    }
+
     /**
      * Enables JPA entity coverage check if {@code enabled} is true. This should only be enabled for
      * members of SqlIntegrationTestSuite.
@@ -221,10 +236,10 @@ public final class AppEngineRule extends ExternalResource
 
     public AppEngineRule build() {
       checkState(
-          !rule.enableJpaEntityCoverageCheck || rule.withDatastoreAndCloudSql,
+          !rule.enableJpaEntityCoverageCheck || rule.withCloudSql,
           "withJpaEntityCoverageCheck enabled without Cloud SQL");
       checkState(
-          !rule.withJpaUnitTest || rule.withDatastoreAndCloudSql,
+          !rule.withJpaUnitTest || rule.withCloudSql,
           "withJpaUnitTestEntities enabled without Cloud SQL");
       checkState(
           !rule.withJpaUnitTest || !rule.enableJpaEntityCoverageCheck,
@@ -341,7 +356,7 @@ public final class AppEngineRule extends ExternalResource
   @Override
   public void beforeEach(ExtensionContext context) throws Exception {
     before();
-    if (withDatastoreAndCloudSql) {
+    if (withCloudSql) {
       JpaTestRules.Builder builder = new JpaTestRules.Builder();
       if (clock != null) {
         builder.withClock(clock);
@@ -360,13 +375,15 @@ public final class AppEngineRule extends ExternalResource
         jpaIntegrationTestRule.before();
       }
     }
-    injectTmForDualDatabaseTest(context);
+    if (isWithDatastoreAndCloudSql()) {
+      injectTmForDualDatabaseTest(context);
+    }
   }
 
   /** Called after each test method. JUnit 5 only. */
   @Override
   public void afterEach(ExtensionContext context) throws Exception {
-    if (withDatastoreAndCloudSql) {
+    if (withCloudSql) {
       if (enableJpaEntityCoverageCheck) {
         jpaIntegrationWithCoverageExtension.afterEach(context);
       } else if (withJpaUnitTest) {
@@ -376,7 +393,9 @@ public final class AppEngineRule extends ExternalResource
       }
     }
     after();
-    restoreTmAfterDualDatabaseTest(context);
+    if (isWithDatastoreAndCloudSql()) {
+      restoreTmAfterDualDatabaseTest(context);
+    }
   }
 
   /**
@@ -389,7 +408,7 @@ public final class AppEngineRule extends ExternalResource
   @Override
   public Statement apply(Statement base, Description description) {
     Statement statement = base;
-    if (withDatastoreAndCloudSql) {
+    if (withCloudSql) {
       JpaTestRules.Builder builder = new JpaTestRules.Builder();
       if (clock != null) {
         builder.withClock(clock);
@@ -410,7 +429,7 @@ public final class AppEngineRule extends ExternalResource
     if (withUrlFetch) {
       configs.add(new LocalURLFetchServiceTestConfig());
     }
-    if (withDatastoreAndCloudSql) {
+    if (withDatastore) {
       configs.add(new LocalDatastoreServiceTestConfig()
           // We need to set this to allow cross entity group transactions.
           .setApplyAllHighRepJobPolicy()
@@ -462,11 +481,13 @@ public final class AppEngineRule extends ExternalResource
 
     helper.setUp();
 
-    if (withDatastoreAndCloudSql) {
+    if (withDatastore) {
       ObjectifyService.initOfy();
       // Reset id allocation in ObjectifyService so that ids are deterministic in tests.
       ObjectifyService.resetNextTestId();
-      loadInitialData();
+      if (!withoutCannedData) {
+        loadInitialData();
+      }
       this.ofyTestEntities.forEach(AppEngineRule::register);
     }
   }
@@ -593,6 +614,6 @@ public final class AppEngineRule extends ExternalResource
   }
 
   boolean isWithDatastoreAndCloudSql() {
-    return withDatastoreAndCloudSql;
+    return withDatastore && withCloudSql;
   }
 }

core/src/test/java/google/registry/testing/DatastoreHelper.java

@@ -139,7 +139,7 @@ public class DatastoreHelper {
   public static DomainBase newDomainBase(String domainName, HostResource host) {
     return newDomainBase(domainName)
         .asBuilder()
-        .setNameservers(ImmutableSet.of(host.createKey()))
+        .setNameservers(ImmutableSet.of(host.createVKey()))
         .build();
   }
 

core/src/test/java/google/registry/testing/DualDatabaseTestInvocationContextProviderTest.java

@@ -0,0 +1,54 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.testing;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import google.registry.model.ofy.DatastoreTransactionManager;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/**
+ * Test to verify that {@link DualDatabaseTestInvocationContextProvider} extension executes {@link
+ * TestTemplate} test twice with different databases.
+ */
+@DualDatabaseTest
+public class DualDatabaseTestInvocationContextProviderTest {
+
+  private static int datastoreTestCounter = 0;
+  private static int postgresqlTestCounter = 0;
+
+  @RegisterExtension
+  public final AppEngineRule appEngine = AppEngineRule.builder().withDatastoreAndCloudSql().build();
+
+  @TestTemplate
+  void testToUseTransactionManager() {
+    if (tm() instanceof DatastoreTransactionManager) {
+      datastoreTestCounter++;
+    }
+    if (tm() instanceof JpaTransactionManager) {
+      postgresqlTestCounter++;
+    }
+  }
+
+  @AfterAll
+  static void assertEachTransactionManagerIsUsed() {
+    assertThat(datastoreTestCounter).isEqualTo(1);
+    assertThat(postgresqlTestCounter).isEqualTo(1);
+  }
+}

core/src/test/java/google/registry/testing/FullFieldsTestEntityHelper.java

@@ -376,10 +376,10 @@ public final class FullFieldsTestEntityHelper {
     if ((ns1 != null) || (ns2 != null)) {
       ImmutableSet.Builder<VKey<HostResource>> nsBuilder = new ImmutableSet.Builder<>();
       if (ns1 != null) {
-        nsBuilder.add(ns1.createKey());
+        nsBuilder.add(ns1.createVKey());
       }
       if (ns2 != null) {
-        nsBuilder.add(ns2.createKey());
+        nsBuilder.add(ns2.createVKey());
       }
       builder.setNameservers(nsBuilder.build());
     }

core/src/test/java/google/registry/tools/GenerateDnsReportCommandTest.java

@@ -147,7 +147,7 @@ public class GenerateDnsReportCommandTest extends CommandTestCase<GenerateDnsRep
         persistResource(
             newDomainBase("example.xn--q9jyb4c")
                 .asBuilder()
-                .setNameservers(ImmutableSet.of(nameserver1.createKey(), nameserver2.createKey()))
+                .setNameservers(ImmutableSet.of(nameserver1.createVKey(), nameserver2.createVKey()))
                 .setDsData(
                     ImmutableSet.of(
                         DelegationSignerData.create(
@@ -158,7 +158,7 @@ public class GenerateDnsReportCommandTest extends CommandTestCase<GenerateDnsRep
     persistResource(
         newDomainBase("foobar.xn--q9jyb4c")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(nameserver3.createKey(), nameserver4.createKey()))
+            .setNameservers(ImmutableSet.of(nameserver3.createVKey(), nameserver4.createVKey()))
             .build());
     // Persist a domain in a different tld that should be ignored.
     persistActiveDomain("should-be-ignored.example");

core/src/test/java/google/registry/tools/LevelDbFileBuilder.java

@@ -39,7 +39,7 @@ public final class LevelDbFileBuilder {
   }
 
   /** Adds an {@link Entity Datastore Entity object} to the leveldb log file. */
-  LevelDbFileBuilder addEntity(Entity entity) throws IOException {
+  public LevelDbFileBuilder addEntity(Entity entity) throws IOException {
     EntityProto proto = EntityTranslator.convertToPb(entity);
     byte[] protoBytes = proto.toByteArray();
     if (protoBytes.length > BLOCK_SIZE - (currentPos + HEADER_SIZE)) {
@@ -53,7 +53,7 @@ public final class LevelDbFileBuilder {
   }
 
   /** Writes all remaining data and closes the block. */
-  void build() throws IOException {
+  public void build() throws IOException {
     out.write(currentBlock);
     out.close();
   }

core/src/test/java/google/registry/tools/UniformRapidSuspensionCommandTest.java

@@ -55,7 +55,7 @@ public class UniformRapidSuspensionCommandTest
   private void persistDomainWithHosts(HostResource... hosts) {
     ImmutableSet.Builder<VKey<HostResource>> hostRefs = new ImmutableSet.Builder<>();
     for (HostResource host : hosts) {
-      hostRefs.add(host.createKey());
+      hostRefs.add(host.createVKey());
     }
     persistResource(newDomainBase("evil.tld").asBuilder()
         .setNameservers(hostRefs.build())

core/src/test/java/google/registry/tools/UpdateDomainCommandTest.java

@@ -116,12 +116,12 @@ public class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomain
     persistResource(
         newDomainBase("example.abc")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host1.createKey()))
+            .setNameservers(ImmutableSet.of(host1.createVKey()))
             .build());
     persistResource(
         newDomainBase("example.tld")
             .asBuilder()
-            .setNameservers(ImmutableSet.of(host2.createKey()))
+            .setNameservers(ImmutableSet.of(host2.createVKey()))
             .build());
     runCommandForced(
         "--client=NewRegistrar", nsParam, "example.abc", "example.tld");
@@ -172,7 +172,7 @@ public class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomain
     HostResource host1 = persistActiveHost("ns1.zdns.google");
     HostResource host2 = persistActiveHost("ns2.zdns.google");
     ImmutableSet<VKey<HostResource>> nameservers =
-        ImmutableSet.of(host1.createKey(), host2.createKey());
+        ImmutableSet.of(host1.createVKey(), host2.createVKey());
     persistResource(
         newDomainBase("example.tld").asBuilder().setNameservers(nameservers).build());
     runCommandForced(
@@ -213,7 +213,7 @@ public class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomain
   @Test
   public void testSuccess_setStatuses() throws Exception {
     HostResource host = persistActiveHost("ns1.zdns.google");
-    ImmutableSet<VKey<HostResource>> nameservers = ImmutableSet.of(host.createKey());
+    ImmutableSet<VKey<HostResource>> nameservers = ImmutableSet.of(host.createVKey());
     persistResource(
         newDomainBase("example.tld")
             .asBuilder()
@@ -257,7 +257,7 @@ public class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomain
   @Test
   public void testFailure_cantUpdateRegistryLockedDomainEvenAsSuperuser() {
     HostResource host = persistActiveHost("ns1.zdns.google");
-    ImmutableSet<VKey<HostResource>> nameservers = ImmutableSet.of(host.createKey());
+    ImmutableSet<VKey<HostResource>> nameservers = ImmutableSet.of(host.createVKey());
     persistResource(
         newDomainBase("example.tld")
             .asBuilder()

core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java

@@ -68,7 +68,7 @@ public class GenerateZoneFilesActionTest extends MapreduceTestCase<GenerateZoneF
         persistResource(newHostResource("ns.bar.tld").asBuilder().addInetAddresses(ips).build());
 
     ImmutableSet<VKey<HostResource>> nameservers =
-        ImmutableSet.of(host1.createKey(), host2.createKey());
+        ImmutableSet.of(host1.createVKey(), host2.createVKey());
     // This domain will have glue records, because it has a subordinate host which is its own
     // nameserver. None of the other domains should have glue records, because their nameservers are
     // subordinate to different domains.

core/src/test/java/google/registry/whois/DomainWhoisResponseTest.java

@@ -220,8 +220,8 @@ public class DomainWhoisResponseTest {
         .setEmailAddress("EMAIL@EXAMPLE.tld")
         .build());
 
-    VKey<HostResource> hostResource1Key = hostResource1.createKey();
-    VKey<HostResource> hostResource2Key = hostResource2.createKey();
+    VKey<HostResource> hostResource1Key = hostResource1.createVKey();
+    VKey<HostResource> hostResource2Key = hostResource2.createVKey();
     VKey<ContactResource> registrantResourceKey = registrant.createVKey();
     VKey<ContactResource> adminResourceKey = adminContact.createVKey();
     VKey<ContactResource> techResourceKey = techContact.createVKey();

core/src/test/resources/google/registry/model/schema.txt

@@ -314,20 +314,20 @@ class google.registry.model.index.EppResourceIndexBucket {
 }
 class google.registry.model.index.ForeignKeyIndex$ForeignKeyContactIndex {
   @Id java.lang.String foreignKey;
-  com.googlecode.objectify.Key<E> topReference;
   google.registry.model.UpdateAutoTimestamp updateTimestamp;
+  google.registry.persistence.VKey<E> topReference;
   org.joda.time.DateTime deletionTime;
 }
 class google.registry.model.index.ForeignKeyIndex$ForeignKeyDomainIndex {
   @Id java.lang.String foreignKey;
-  com.googlecode.objectify.Key<E> topReference;
   google.registry.model.UpdateAutoTimestamp updateTimestamp;
+  google.registry.persistence.VKey<E> topReference;
   org.joda.time.DateTime deletionTime;
 }
 class google.registry.model.index.ForeignKeyIndex$ForeignKeyHostIndex {
   @Id java.lang.String foreignKey;
-  com.googlecode.objectify.Key<E> topReference;
   google.registry.model.UpdateAutoTimestamp updateTimestamp;
+  google.registry.persistence.VKey<E> topReference;
   org.joda.time.DateTime deletionTime;
 }
 class google.registry.model.ofy.CommitLogBucket {
@@ -373,14 +373,14 @@ class google.registry.model.poll.PendingActionNotificationResponse$NameOrId {
   java.lang.String value;
 }
 class google.registry.model.poll.PollMessage {
-  @Id long id;
+  @Id java.lang.Long id;
   @Parent com.googlecode.objectify.Key<google.registry.model.reporting.HistoryEntry> parent;
   java.lang.String clientId;
   java.lang.String msg;
   org.joda.time.DateTime eventTime;
 }
 class google.registry.model.poll.PollMessage$Autorenew {
-  @Id long id;
+  @Id java.lang.Long id;
   @Parent com.googlecode.objectify.Key<google.registry.model.reporting.HistoryEntry> parent;
   java.lang.String clientId;
   java.lang.String msg;
@@ -389,7 +389,7 @@ class google.registry.model.poll.PollMessage$Autorenew {
   org.joda.time.DateTime eventTime;
 }
 class google.registry.model.poll.PollMessage$OneTime {
-  @Id long id;
+  @Id java.lang.Long id;
   @Parent com.googlecode.objectify.Key<google.registry.model.reporting.HistoryEntry> parent;
   java.lang.String clientId;
   java.lang.String msg;

core/src/test/resources/google/registry/reporting/spec11/spec11_fake_report

@@ -1,3 +1,3 @@
 Map from registrar email / name to detected subdomain threats:
-{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"a.com","platformType":"ANY_PLATFORM"}],"registrarClientId":"TheRegistrar","registrarEmailAddress":"the.registrar@example.com"}
-{"threatMatches":[{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"b.com","platformType":"ANY_PLATFORM"},{"threatEntryMetadata":"NONE","threatType":"MALWARE","fullyQualifiedDomainName":"c.com","platformType":"ANY_PLATFORM"}],"registrarClientId":"NewRegistrar","registrarEmailAddress":"new.registrar@example.com"}
+{"threatMatches":[{"threatType":"MALWARE","fullyQualifiedDomainName":"a.com"}],"registrarClientId":"TheRegistrar","registrarEmailAddress":"the.registrar@example.com"}
+{"threatMatches":[{"threatType":"MALWARE","fullyQualifiedDomainName":"b.com"},{"threatType":"MALWARE","fullyQualifiedDomainName":"c.com"}],"registrarClientId":"NewRegistrar","registrarEmailAddress":"new.registrar@example.com"}

db/src/main/resources/sql/flyway/V27__create_pollmessage.sql

@@ -0,0 +1,76 @@
+-- Copyright 2020 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+create table "PollMessage" (
+    type text not null,
+    poll_message_id  bigserial not null,
+    registrar_id text not null,
+    contact_repo_id text,
+    contact_revision_id int8,
+    domain_repo_id text,
+    domain_revision_id int8,
+    event_time timestamptz not null,
+    host_repo_id text,
+    host_revision_id int8,
+    message text,
+    transfer_response_contact_id text,
+    transfer_response_domain_expiration_time timestamptz,
+    transfer_response_domain_name text,
+    pending_action_response_action_result boolean,
+    pending_action_response_name_or_id text,
+    pending_action_response_processed_date timestamptz,
+    pending_action_response_client_txn_id text,
+    pending_action_response_server_txn_id text,
+    transfer_response_gaining_registrar_id text,
+    transfer_response_losing_registrar_id text,
+    transfer_response_pending_transfer_expiration_time timestamptz,
+    transfer_response_transfer_request_time timestamptz,
+    transfer_response_transfer_status text,
+    autorenew_end_time timestamptz,
+    autorenew_domain_name text,
+    primary key (poll_message_id)
+);
+
+create index IDXe7wu46c7wpvfmfnj4565abibp on "PollMessage" (registrar_id);
+create index IDXaydgox62uno9qx8cjlj5lauye on "PollMessage" (event_time);
+
+alter table if exists "PollMessage"
+   add constraint fk_poll_message_registrar_id
+   foreign key (registrar_id)
+   references "Registrar";
+
+alter table if exists "PollMessage"
+   add constraint fk_poll_message_contact_repo_id
+   foreign key (contact_repo_id)
+   references "Contact";
+
+alter table if exists "PollMessage"
+   add constraint fk_poll_message_domain_repo_id
+   foreign key (domain_repo_id)
+   references "Domain";
+
+alter table if exists "PollMessage"
+   add constraint fk_poll_message_host_repo_id
+   foreign key (host_repo_id)
+   references "HostResource";
+
+alter table if exists "PollMessage"
+   add constraint fk_poll_message_transfer_response_gaining_registrar_id
+   foreign key (transfer_response_gaining_registrar_id)
+   references "Registrar";
+
+alter table if exists "PollMessage"
+   add constraint fk_poll_message_transfer_response_losing_registrar_id
+   foreign key (transfer_response_losing_registrar_id)
+   references "Registrar";

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -213,6 +213,36 @@
         primary key (resource_name, tld)
     );
 
+    create table "PollMessage" (
+       type text not null,
+        poll_message_id  bigserial not null,
+        registrar_id text not null,
+        contact_repo_id text,
+        contact_revision_id int8,
+        domain_repo_id text,
+        domain_revision_id int8,
+        event_time timestamptz not null,
+        host_repo_id text,
+        host_revision_id int8,
+        message text,
+        transfer_response_contact_id text,
+        transfer_response_domain_expiration_time timestamptz,
+        transfer_response_domain_name text,
+        pending_action_response_action_result boolean,
+        pending_action_response_name_or_id text,
+        pending_action_response_processed_date timestamptz,
+        pending_action_response_client_txn_id text,
+        pending_action_response_server_txn_id text,
+        transfer_response_gaining_registrar_id text,
+        transfer_response_losing_registrar_id text,
+        transfer_response_pending_transfer_expiration_time timestamptz,
+        transfer_response_transfer_request_time timestamptz,
+        transfer_response_transfer_status text,
+        autorenew_end_time timestamptz,
+        autorenew_domain_name text,
+        primary key (poll_message_id)
+    );
+
     create table "PremiumEntry" (
        revision_id int8 not null,
         domain_label text not null,
@@ -352,6 +382,8 @@ create index IDX8ffrqm27qtj20jac056j7yq07 on "Domain" (current_sponsor_client_id
 create index IDX5mnf0wn20tno4b9do88j61klr on "Domain" (deletion_time);
 create index IDX1rcgkdd777bpvj0r94sltwd5y on "Domain" (fully_qualified_domain_name);
 create index IDXrwl38wwkli1j7gkvtywi9jokq on "Domain" (tld);
+create index IDXe7wu46c7wpvfmfnj4565abibp on "PollMessage" (registrar_id);
+create index IDXaydgox62uno9qx8cjlj5lauye on "PollMessage" (event_time);
 create index premiumlist_name_idx on "PremiumList" (name);
 create index registrar_name_idx on "Registrar" (registrar_name);
 create index registrar_iana_identifier_idx on "Registrar" (iana_identifier);

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -343,6 +343,59 @@ CREATE TABLE public."Lock" (
 );
 
 
+--
+-- Name: PollMessage; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public."PollMessage" (
+    type text NOT NULL,
+    poll_message_id bigint NOT NULL,
+    registrar_id text NOT NULL,
+    contact_repo_id text,
+    contact_revision_id bigint,
+    domain_repo_id text,
+    domain_revision_id bigint,
+    event_time timestamp with time zone NOT NULL,
+    host_repo_id text,
+    host_revision_id bigint,
+    message text,
+    transfer_response_contact_id text,
+    transfer_response_domain_expiration_time timestamp with time zone,
+    transfer_response_domain_name text,
+    pending_action_response_action_result boolean,
+    pending_action_response_name_or_id text,
+    pending_action_response_processed_date timestamp with time zone,
+    pending_action_response_client_txn_id text,
+    pending_action_response_server_txn_id text,
+    transfer_response_gaining_registrar_id text,
+    transfer_response_losing_registrar_id text,
+    transfer_response_pending_transfer_expiration_time timestamp with time zone,
+    transfer_response_transfer_request_time timestamp with time zone,
+    transfer_response_transfer_status text,
+    autorenew_end_time timestamp with time zone,
+    autorenew_domain_name text
+);
+
+
+--
+-- Name: PollMessage_poll_message_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public."PollMessage_poll_message_id_seq"
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: PollMessage_poll_message_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public."PollMessage_poll_message_id_seq" OWNED BY public."PollMessage".poll_message_id;
+
+
 --
 -- Name: PremiumEntry; Type: TABLE; Schema: public; Owner: -
 --
@@ -572,6 +625,13 @@ ALTER TABLE ONLY public."BillingRecurrence" ALTER COLUMN billing_recurrence_id S
 ALTER TABLE ONLY public."ClaimsList" ALTER COLUMN revision_id SET DEFAULT nextval('public."ClaimsList_revision_id_seq"'::regclass);
 
 
+--
+-- Name: PollMessage poll_message_id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."PollMessage" ALTER COLUMN poll_message_id SET DEFAULT nextval('public."PollMessage_poll_message_id_seq"'::regclass);
+
+
 --
 -- Name: PremiumList revision_id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -673,6 +733,14 @@ ALTER TABLE ONLY public."Lock"
     ADD CONSTRAINT "Lock_pkey" PRIMARY KEY (resource_name, tld);
 
 
+--
+-- Name: PollMessage PollMessage_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."PollMessage"
+    ADD CONSTRAINT "PollMessage_pkey" PRIMARY KEY (poll_message_id);
+
+
 --
 -- Name: PremiumEntry PremiumEntry_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -829,6 +897,13 @@ CREATE INDEX idx_registry_lock_registrar_id ON public."RegistryLock" USING btree
 CREATE INDEX idx_registry_lock_verification_code ON public."RegistryLock" USING btree (verification_code);
 
 
+--
+-- Name: idxaydgox62uno9qx8cjlj5lauye; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxaydgox62uno9qx8cjlj5lauye ON public."PollMessage" USING btree (event_time);
+
+
 --
 -- Name: idxbn8t4wp85fgxjl8q4ctlscx55; Type: INDEX; Schema: public; Owner: -
 --
@@ -836,6 +911,13 @@ CREATE INDEX idx_registry_lock_verification_code ON public."RegistryLock" USING
 CREATE INDEX idxbn8t4wp85fgxjl8q4ctlscx55 ON public."Contact" USING btree (current_sponsor_client_id);
 
 
+--
+-- Name: idxe7wu46c7wpvfmfnj4565abibp; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxe7wu46c7wpvfmfnj4565abibp ON public."PollMessage" USING btree (registrar_id);
+
+
 --
 -- Name: idxeokttmxtpq2hohcioe5t2242b; Type: INDEX; Schema: public; Owner: -
 --
@@ -1085,6 +1167,54 @@ ALTER TABLE ONLY public."DomainHost"
     ADD CONSTRAINT fk_domainhost_host_valid FOREIGN KEY (ns_hosts) REFERENCES public."HostResource"(repo_id);
 
 
+--
+-- Name: PollMessage fk_poll_message_contact_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."PollMessage"
+    ADD CONSTRAINT fk_poll_message_contact_repo_id FOREIGN KEY (contact_repo_id) REFERENCES public."Contact"(repo_id);
+
+
+--
+-- Name: PollMessage fk_poll_message_domain_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."PollMessage"
+    ADD CONSTRAINT fk_poll_message_domain_repo_id FOREIGN KEY (domain_repo_id) REFERENCES public."Domain"(repo_id);
+
+
+--
+-- Name: PollMessage fk_poll_message_host_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."PollMessage"
+    ADD CONSTRAINT fk_poll_message_host_repo_id FOREIGN KEY (host_repo_id) REFERENCES public."HostResource"(repo_id);
+
+
+--
+-- Name: PollMessage fk_poll_message_registrar_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."PollMessage"
+    ADD CONSTRAINT fk_poll_message_registrar_id FOREIGN KEY (registrar_id) REFERENCES public."Registrar"(client_id);
+
+
+--
+-- Name: PollMessage fk_poll_message_transfer_response_gaining_registrar_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."PollMessage"
+    ADD CONSTRAINT fk_poll_message_transfer_response_gaining_registrar_id FOREIGN KEY (transfer_response_gaining_registrar_id) REFERENCES public."Registrar"(client_id);
+
+
+--
+-- Name: PollMessage fk_poll_message_transfer_response_losing_registrar_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."PollMessage"
+    ADD CONSTRAINT fk_poll_message_transfer_response_losing_registrar_id FOREIGN KEY (transfer_response_losing_registrar_id) REFERENCES public."Registrar"(client_id);
+
+
 --
 -- Name: DomainHost fkfmi7bdink53swivs390m2btxg; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

release/cloudbuild-javadoc.yaml

@@ -0,0 +1,30 @@
+# To run the build locally, install cloud-build-local first.
+# See: https://cloud.google.com/cloud-build/docs/build-debug-locally
+# In the root of a nomulus source tree, run:
+# cloud-build-local --config=cloudbuild-javadoc.yaml --dryrun=false ..
+#
+# This will compile javadoc for the FOSS version of the code base and replace
+# the content at gs://${PROJECT_ID}-javadoc with the it. The compiled javadoc
+# can then be accesssed at https://storage.googleapis.com/${PROJECT_ID}-javadoc
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config cloudbuild-javadoc.yaml ..
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+steps:
+# Compile javadoc
+- name: 'gcr.io/${PROJECT_ID}/builder'
+  entrypoint: /bin/bash
+  args: ['./gradlew', ':javadoc']
+# Upload the files to GCS
+# We don't use GCB's built-in artifacts uploader because we want to delete
+# the existing files in the bucket first, and we want to parallelize the
+# uploading process.
+- name: 'gcr.io/${PROJECT_ID}/builder'
+  entrypoint: /bin/bash
+  args: ['gsutil', '-m', 'rsync', '-d', '-r', 'build/docs/javadoc', 'gs://${PROJECT_ID}-javadoc']
+
+timeout: 3600s
+options:
+  machineType: 'N1_HIGHCPU_8'

release/cloudbuild-kythe.yaml

@@ -2,13 +2,13 @@
 # See: https://cloud.google.com/cloud-build/docs/build-debug-locally
 # In the root of a nomulus source tree, run:
 # cloud-build-local --config=cloudbuild-kythe.yaml --dryrun=false \
-#   --substitutions _KYTHE_VERSION=[kythe_version],COMMIT_HASH=[hash] .
+#   --substitutions _KYTHE_VERSION=[kythe_version],COMMIT_SHA=[hash] ..
 # This will download kythe version ${kythe_version} (must be higher than
 # v0.0.39 and build a ${hash}.kzip file for Kythe to enable cross referencing.
 #
 # To manually trigger a build on GCB, run:
 # gcloud builds submit --config cloudbuild-kythe.yaml --substitutions \
-#   _KYTHE_VERSION[kythe_version] .
+#   _KYTHE_VERSION=[kythe_version] ..
 #
 # To trigger a build automatically, follow the instructions below and add a trigger:
 # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds