Add an extension to verify transaction replay (#857)

* Add an extension to verify transaction replay

Add ReplayExtension, which can be applied to test suites to verify that
transactions committed to datastore can be replayed to SQL.

This introduces a ReplayQueue class, which serves as a stand-in for the
current lack of replay-from-commit-logs.  It also includes replay logic in
TransactionInfo which introduces the concept of "entity class weights."
Entity weighting allows us store and delete objects in an order that is
consistent with the direction of foreign key and deferred foreign key
relationships.  As a general rule, lower weight classes must have no direct or
indirect non-deferred foreign key relationships on higher weight classes.

It is expected that much of this code will change when the final replay
mechanism is implemented.

* Minor fixes:

- Initialize "requestedByRegistrar" to false (it's non-nullable). [reverted
  during rebase: non-nullable was removed in another PR]
- Store test entities (registrar, hosts and contacts) in JPA.

* Make testbed save replay

This changes the replay system to make datastore saves initiated from the
testbed (as opposed to just the tested code) replay when the ReplayExtension
is enabled.  This requires modifications to DatastoreHelper and the
AppEngineExtension that the ReplayExtension can plug into.

This changes also has some necessary fixes to objects that are persisted by
the testbed (such as PremiumList).

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -57,6 +57,7 @@ import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.persistence.Table;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /** An entity representing an allocation token. */
@@ -105,7 +106,8 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
   @javax.persistence.Id @Id String token;
 
   /** The key of the history entry for which the token was used. Null if not yet used. */
-  @Nullable @Index VKey<HistoryEntry> redemptionHistoryEntry;
+  // TODO(b/172848495): Remove the "Transient" when we can finally persist and restore this.
+  @Transient @Nullable @Index VKey<HistoryEntry> redemptionHistoryEntry;
 
   /** The fully-qualified domain name that this token is limited to, if any. */
   @Nullable @Index String domainName;

core/src/main/java/google/registry/model/ofy/CommitLoggedWork.java

@@ -161,6 +161,7 @@ class CommitLoggedWork<R> implements Runnable {
           .addAll(untouchedRootsWithTouchedChildren)
           .build())
       .now();
+    ReplayQueue.addInTests(info);
   }
 
   /** Check that the timestamp of each BackupGroupRoot is in the past. */

core/src/main/java/google/registry/model/ofy/ReplayQueue.java

@@ -0,0 +1,46 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.ofy;
+
+import google.registry.config.RegistryEnvironment;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * Implements simplified datastore to SQL transaction replay.
+ *
+ * <p>This code is to be removed when the actual replay cron job is implemented.
+ */
+public class ReplayQueue {
+
+  static ConcurrentLinkedQueue<TransactionInfo> queue =
+      new ConcurrentLinkedQueue<TransactionInfo>();
+
+  static void addInTests(TransactionInfo info) {
+    if (RegistryEnvironment.get() == RegistryEnvironment.UNITTEST) {
+      queue.add(info);
+    }
+  }
+
+  public static void replay() {
+    TransactionInfo info;
+    while ((info = queue.poll()) != null) {
+      info.saveToJpa();
+    }
+  }
+
+  public static void clear() {
+    queue.clear();
+  }
+}

core/src/main/java/google/registry/model/ofy/TransactionInfo.java

@@ -21,17 +21,25 @@ import static com.google.common.collect.Maps.filterValues;
 import static com.google.common.collect.Maps.toMap;
 import static google.registry.model.ofy.CommitLogBucket.getArbitraryBucketId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
+import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.Map;
 import org.joda.time.DateTime;
 
 /** Metadata for an {@link Ofy} transaction that saves commit logs. */
 class TransactionInfo {
 
-  private enum Delete { SENTINEL }
+  @VisibleForTesting
+  enum Delete {
+    SENTINEL
+  }
 
   /** Logical "now" of the transaction. */
   DateTime transactionTime;
@@ -92,4 +100,49 @@ class TransactionInfo {
         .filter(not(Delete.SENTINEL::equals))
         .collect(toImmutableSet());
   }
+
+  // Mapping from class name to "weight" (which in this case is the order in which the class must
+  // be "put" in a transaction with respect to instances of other classes).  Lower weight classes
+  // are put first, by default all classes have a weight of zero.
+  static final ImmutableMap<String, Integer> CLASS_WEIGHTS =
+      ImmutableMap.of(
+          "HistoryEntry", -1,
+          "DomainBase", 1);
+
+  // The beginning of the range of weights reserved for delete.  This must be greater than any of
+  // the values in CLASS_WEIGHTS by enough overhead to accomodate any negative values in it.
+  @VisibleForTesting static final int DELETE_RANGE = Integer.MAX_VALUE / 2;
+
+  /** Returns the weight of the entity type in the map entry. */
+  @VisibleForTesting
+  static int getWeight(ImmutableMap.Entry<Key<?>, Object> entry) {
+    int weight = CLASS_WEIGHTS.getOrDefault(entry.getKey().getKind(), 0);
+    return entry.getValue().equals(Delete.SENTINEL) ? DELETE_RANGE - weight : weight;
+  }
+
+  private static int compareByWeight(
+      ImmutableMap.Entry<Key<?>, Object> a, ImmutableMap.Entry<Key<?>, Object> b) {
+    return getWeight(a) - getWeight(b);
+  }
+
+  void saveToJpa() {
+    // Sort the changes into an order that will work for insertion into the database.
+    jpaTm()
+        .transact(
+            () -> {
+              changesBuilder.build().entrySet().stream()
+                  .sorted(TransactionInfo::compareByWeight)
+                  .forEach(
+                      entry -> {
+                        if (entry.getValue().equals(Delete.SENTINEL)) {
+                          jpaTm().delete(VKey.from(entry.getKey()));
+                        } else {
+                          for (SqlEntity entity :
+                              ((DatastoreEntity) entry.getValue()).toSqlEntities()) {
+                            jpaTm().put(entity);
+                          }
+                        }
+                      });
+            });
+  }
 }

core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java

@@ -79,7 +79,7 @@ public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends Dom
   // set to the timestamp when the list is created. In Datastore, we have two fields and the
   // lastUpdateTime is set to the current timestamp when creating and updating a list. So, we use
   // lastUpdateTime as the creation_timestamp column during the dual-write phase for compatibility.
-  @Column(name = "creation_timestamp", nullable = false)
+  @Column(name = "creation_timestamp")
   DateTime lastUpdateTime;
 
   /** Returns the ID of this revision, or throws if null. */