Add a beforeSqlSave callback to ReplaySpecializer (#1062 )

* Add a beforeSqlSave callback to ReplaySpecializer When in the Datastore-primary and SQL-secondary stage, we will want to save the EppResource-at-this-point-in-time field in the *History objects so that later on we can examine the *History objects to see what the resource looked like at that point in time. Without this PR, the full object at that point in time would be lost during the asynchronous replay since Datastore doesn't know about it. In addition, we modify the HistoryEntry weight / priority so that additions to it come after the additions to the resource off of which it is based. As a result, we need to DEFER some foreign keys so that we can write the billing / poll message objects before the history object that they're referencing.
Make Numulus compile on macOS (#1070 )
2026-08-14 19:26:09 +00:00 · 2021-04-12 12:11:20 -04:00 · 2021-04-12 10:12:26 -04:00 · 2021-04-08 11:47:35 -04:00
16 changed files with 224 additions and 98 deletions
@@ -423,7 +423,7 @@ task jaxbToJava {
      }
    }
    execInBash(
-        'find . -name *.java -exec sed -i /\\*\\ \\<p\\>\\$/d {} +',
+        "find . -name *.java -exec sed -i -e '/" + /\* <p>$/ + "/d' {} +",
        generatedDir)
  }
 }
@@ -154,7 +154,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
    Object ofyPojo = ofy().toPojo(entity);
    if (ofyPojo instanceof DatastoreEntity) {
      DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
-      datastoreEntity.toSqlEntity().ifPresent(jpaTm()::put);
+      datastoreEntity
+          .toSqlEntity()
+          .ifPresent(
+              sqlEntity -> {
+                ReplaySpecializer.beforeSqlSave(sqlEntity);
+                jpaTm().put(sqlEntity);
+              });
    } else {
      // this should never happen, but we shouldn't fail on it
      logger.atSevere().log(
@@ -14,6 +14,8 @@

 package google.registry.model.contact;

+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -114,6 +116,12 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
    return Optional.of(asHistoryEntry());
  }

+  // Used to fill out the contactBase field during asynchronous replay
+  public static void beforeSqlSave(ContactHistory contactHistory) {
+    contactHistory.contactBase =
+        jpaTm().loadByKey(VKey.createSql(ContactResource.class, contactHistory.getContactRepoId()));
+  }
+
  /** Class to represent the composite primary key of {@link ContactHistory} entity. */
  public static class ContactHistoryId extends ImmutableObject implements Serializable {

@@ -15,6 +15,7 @@
 package google.registry.model.domain;

 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;

 import com.google.common.collect.ImmutableSet;
@@ -263,6 +264,12 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
    return Optional.of(asHistoryEntry());
  }

+  // Used to fill out the domainContent field during asynchronous replay
+  public static void beforeSqlSave(DomainHistory domainHistory) {
+    domainHistory.domainContent =
+        jpaTm().loadByKey(VKey.createSql(DomainBase.class, domainHistory.getDomainRepoId()));
+  }
+
  /** Class to represent the composite primary key of {@link DomainHistory} entity. */
  public static class DomainHistoryId extends ImmutableObject implements Serializable {

@@ -14,6 +14,8 @@

 package google.registry.model.host;

+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -115,6 +117,12 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
    return Optional.of(asHistoryEntry());
  }

+  // Used to fill out the hostBase field during asynchronous replay
+  public static void beforeSqlSave(HostHistory hostHistory) {
+    hostHistory.hostBase =
+        jpaTm().loadByKey(VKey.createSql(HostResource.class, hostHistory.getHostRepoId()));
+  }
+
  /** Class to represent the composite primary key of {@link HostHistory} entity. */
  public static class HostHistoryId extends ImmutableObject implements Serializable {

@@ -41,10 +41,11 @@ public class EntityWritePriorities {
   */
  static final ImmutableMap<String, Integer> CLASS_PRIORITIES =
      ImmutableMap.of(
-          "ContactResource", -15,
-          "HistoryEntry", -10,
          "AllocationToken", -9,
-          "DomainBase", 10);
+          "ContactResource", 8,
+          "HostResource", 9,
+          "DomainBase", 10,
+          "HistoryEntry", 20);

  // The beginning of the range of priority numbers reserved for delete.  This must be greater than
  // any of the values in CLASS_PRIORITIES by enough overhead to accommodate  any negative values in
@@ -26,21 +26,30 @@ import java.lang.reflect.Method;
 * to invoke special class methods if they are present.
 */
 public class ReplaySpecializer {
+
  public static void beforeSqlDelete(VKey<?> key) {
+    invokeMethod(key.getKind(), "beforeSqlDelete", key);
+  }
+
+  public static void beforeSqlSave(SqlEntity sqlEntity) {
+    invokeMethod(sqlEntity.getClass(), "beforeSqlSave", sqlEntity);
+  }
+
+  private static <T> void invokeMethod(Class<T> clazz, String methodName, Object argument) {
    try {
-      Method method = key.getKind().getMethod("beforeSqlDelete", VKey.class);
-      method.invoke(null, new Object[] {key});
+      Method method = clazz.getMethod(methodName, argument.getClass());
+      method.invoke(null, argument);
    } catch (NoSuchMethodException e) {
      // Ignore, this just means that the class doesn't need this hook.
    } catch (IllegalAccessException e) {
      throw new RuntimeException(
-          "beforeSqlDelete() method is defined for class "
-              + key.getKind().getName()
-              + " but is not public.",
+          String.format(
+              "%s() method is defined for class %s but is not public.",
+              methodName, clazz.getName()),
          e);
    } catch (InvocationTargetException e) {
      throw new RuntimeException(
-          "beforeSqlDelete() method for class " + key.getKind().getName() + " threw an exception.",
+          String.format("%s() method for class %s threw an exception", methodName, clazz.getName()),
          e);
    }
  }
@@ -125,6 +125,7 @@ public class ReplayCommitLogsToSqlActionTest {
    action.diffLister.gcsBucket = GCS_BUCKET;
    action.diffLister.executor = newDirectExecutorService();
    RegistryConfig.overrideCloudSqlReplayCommitLogs(true);
+    TestObject.beforeSqlSaveCallCount = 0;
    TestObject.beforeSqlDeleteCallCount = 0;
  }

@@ -442,6 +443,21 @@ public class ReplayCommitLogsToSqlActionTest {
        .isEqualTo("Can't acquire SQL commit log replay lock, aborting.");
  }

+  @Test
+  void testSuccess_beforeSqlSaveCallback() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    Key<CommitLogBucket> bucketKey = getBucketKey(1);
+    Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(bucketKey, now, null),
+        CommitLogMutation.create(manifestKey, TestObject.create("a")));
+    runAndAssertSuccess(now.minusMinutes(1));
+    assertThat(TestObject.beforeSqlSaveCallCount).isEqualTo(1);
+  }
+
  @Test
  void testSuccess_deleteSqlCallback() throws Exception {
    DateTime now = fakeClock.nowUtc();
@@ -16,6 +16,7 @@ package google.registry.batch;

 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
+import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_CREATE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
@@ -121,14 +122,30 @@ class DeleteExpiredDomainsActionTest {
  }

  @Test
-  void test_deletesThreeDomainsInOneRun() {
+  void test_deletesThreeDomainsInOneRun() throws Exception {
    DomainBase domain1 = persistNonAutorenewingDomain("ecck1.tld");
    DomainBase domain2 = persistNonAutorenewingDomain("veee2.tld");
    DomainBase domain3 = persistNonAutorenewingDomain("tarm3.tld");

-    // action.run() executes a non-transactional query by design but makes this test flaky.
-    // Executing a transaction here seems to force the test Datastore to become up to date.
-    assertThat(tm().loadByEntity(domain3).getStatusValues()).doesNotContain(PENDING_DELETE);
+    // action.run() executes an ancestor-less query which is subject to eventual consistency (it
+    // uses an index that is updated asynchronously). For a deterministic test outcome, we busy
+    // wait here until all domains above are returned by the query.
+    for (int attempts = 0; attempts < 3; attempts++) {
+      ImmutableSet<String> matchingDomains =
+          ofy()
+              .load()
+              .type(DomainBase.class)
+              .filter("autorenewEndTime <=", clock.nowUtc())
+              .list()
+              .stream()
+              .map(DomainBase::getDomainName)
+              .collect(ImmutableSet.toImmutableSet());
+      if (matchingDomains.containsAll(ImmutableSet.of("ecck1.tld", "veee2.tld", "tarm3.tld"))) {
+        break;
+      }
+      Thread.sleep(100);
+    }
+
    action.run();

    assertThat(tm().loadByEntity(domain1).getStatusValues()).contains(PENDING_DELETE);
@@ -40,7 +40,7 @@ class EntityWritePrioritiesTest {
            Key.create(HistoryEntry.class, 200), "fake history entry",
            Key.create(Registrar.class, 300), "fake registrar");
    ImmutableMap<Long, Integer> expectedValues =
-        ImmutableMap.of(100L, EntityWritePriorities.DELETE_RANGE + 10, 200L, -10, 300L, 0);
+        ImmutableMap.of(100L, EntityWritePriorities.DELETE_RANGE - 20, 200L, 20, 300L, 0);

    for (ImmutableMap.Entry<Key<?>, Object> entry : actions.entrySet()) {
      assertThat(
@@ -34,6 +34,7 @@ import javax.persistence.Transient;
@EntityForTesting
 public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity {

+  public static int beforeSqlSaveCallCount;
  public static int beforeSqlDeleteCallCount;

  @Parent @Transient Key<EntityGroupRoot> parent;
@@ -74,6 +75,10 @@ public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity
    beforeSqlDeleteCallCount++;
  }

+  public static void beforeSqlSave(TestObject testObject) {
+    beforeSqlSaveCallCount++;
+  }
+
  /** A test @VirtualEntity model object, which should not be persisted. */
  @Entity
  @VirtualEntity
@@ -261,11 +261,11 @@ td.section {
    </tr> 
    <tr> 
     <td class="property_name">generated on</td> 
-     <td class="property_value">2021-03-24 01:27:00.824998</td> 
+     <td class="property_value">2021-04-08 17:21:58.993542</td> 
    </tr> 
    <tr>
     <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V90__update_timestamp.sql</td>
+     <td id="lastFlywayFile" class="property_value">V91__defer_fkeys.sql</td>
    </tr>
   </tbody>
  </table> 
@@ -284,7 +284,7 @@ td.section {
     generated on
    </text> 
    <text text-anchor="start" x="4027.94" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2021-03-24 01:27:00.824998
+     2021-04-08 17:21:58.993542
    </text> 
    <polygon fill="none" stroke="#888888" points="3940.44,-4 3940.44,-44 4205.44,-44 4205.44,-4 3940.44,-4" /> <!-- allocationtoken_a08ccbef --> 
    <g id="node1" class="node"> 
@@ -261,32 +261,32 @@ td.section {
    </tr> 
    <tr> 
     <td class="property_name">generated on</td> 
-     <td class="property_value">2021-03-24 01:26:58.684653</td> 
+     <td class="property_value">2021-04-08 17:21:56.891855</td> 
    </tr> 
    <tr>
     <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V90__update_timestamp.sql</td>
+     <td id="lastFlywayFile" class="property_value">V91__defer_fkeys.sql</td>
    </tr>
   </tbody>
  </table> 
  <p>&nbsp;</p> 
  <p>&nbsp;</p> 
-  <svg viewbox="0.00 0.00 4867.18 4862.91" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="erDiagram" style="overflow: hidden; width: 100%; height: 800px"> <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 4858.91)"> 
+  <svg viewbox="0.00 0.00 4850.02 4862.91" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="erDiagram" style="overflow: hidden; width: 100%; height: 800px"> <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 4858.91)"> 
    <title>SchemaCrawler_Diagram</title> 
-    <polygon fill="white" stroke="transparent" points="-4,4 -4,-4858.91 4863.18,-4858.91 4863.18,4 -4,4" /> 
-    <text text-anchor="start" x="4590.68" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
+    <polygon fill="white" stroke="transparent" points="-4,4 -4,-4858.91 4846.02,-4858.91 4846.02,4 -4,4" /> 
+    <text text-anchor="start" x="4573.52" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
     generated by
    </text> 
-    <text text-anchor="start" x="4673.68" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
+    <text text-anchor="start" x="4656.52" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
     SchemaCrawler 16.10.1
    </text> 
-    <text text-anchor="start" x="4589.68" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
+    <text text-anchor="start" x="4572.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
     generated on
    </text> 
-    <text text-anchor="start" x="4673.68" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2021-03-24 01:26:58.684653
+    <text text-anchor="start" x="4656.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
+     2021-04-08 17:21:56.891855
    </text> 
-    <polygon fill="none" stroke="#888888" points="4586.18,-4 4586.18,-44 4851.18,-44 4851.18,-4 4586.18,-4" /> <!-- allocationtoken_a08ccbef --> 
+    <polygon fill="none" stroke="#888888" points="4569.02,-4 4569.02,-44 4834.02,-44 4834.02,-4 4569.02,-4" /> <!-- allocationtoken_a08ccbef --> 
    <g id="node1" class="node"> 
     <title>allocationtoken_a08ccbef</title> 
     <polygon fill="#ebcef2" stroke="transparent" points="3001.5,-2741.91 3001.5,-2760.91 3200.5,-2760.91 3200.5,-2741.91 3001.5,-2741.91" /> 
@@ -5985,142 +5985,142 @@ td.section {
    </g> <!-- registrylock_ac88663e --> 
    <g id="node29" class="node"> 
     <title>registrylock_ac88663e</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4491.5,-4010.91 4491.5,-4029.91 4687.5,-4029.91 4687.5,-4010.91 4491.5,-4010.91" /> 
-     <text text-anchor="start" x="4493.5" y="-4017.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4510.5,-4010.91 4510.5,-4029.91 4669.5,-4029.91 4669.5,-4010.91 4510.5,-4010.91" /> 
+     <text text-anchor="start" x="4512.5" y="-4017.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
      public.RegistryLock
     </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4687.5,-4010.91 4687.5,-4029.91 4813.5,-4029.91 4813.5,-4010.91 4687.5,-4010.91" /> 
-     <text text-anchor="start" x="4774.5" y="-4016.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4669.5,-4010.91 4669.5,-4029.91 4795.5,-4029.91 4795.5,-4010.91 4669.5,-4010.91" /> 
+     <text text-anchor="start" x="4756.5" y="-4016.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      [table]
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3998.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3998.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
      revision_id
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3997.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3997.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3997.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3997.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      bigserial not null
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3978.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3978.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3978.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3978.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      auto-incremented
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      lock_completion_timestamp
+     <text text-anchor="start" x="4512.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      lock_completion_time
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      timestamptz
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      lock_request_timestamp
+     <text text-anchor="start" x="4512.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      lock_request_time
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      timestamptz not null
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      domain_name
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      text not null
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      is_superuser
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      bool not null
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      registrar_id
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      text not null
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      registrar_poc_id
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      text
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      repo_id
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      text not null
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      verification_code
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      text not null
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      unlock_request_timestamp
+     <text text-anchor="start" x="4512.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      unlock_request_time
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      timestamptz
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      unlock_completion_timestamp
+     <text text-anchor="start" x="4512.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      unlock_completion_time
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      timestamptz
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      last_update_time
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      timestamptz not null
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      relock_revision_id
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      int8
     </text> 
-     <text text-anchor="start" x="4493.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      relock_duration
     </text> 
-     <text text-anchor="start" x="4681.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
     </text> 
-     <text text-anchor="start" x="4689.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      interval
     </text> 
-     <polygon fill="none" stroke="#888888" points="4490.5,-3724.91 4490.5,-4030.91 4814.5,-4030.91 4814.5,-3724.91 4490.5,-3724.91" /> 
+     <polygon fill="none" stroke="#888888" points="4509,-3724.91 4509,-4030.91 4796,-4030.91 4796,-3724.91 4509,-3724.91" /> 
    </g> <!-- registrylock_ac88663e&#45;&gt;registrylock_ac88663e --> 
    <g id="edge72" class="edge"> 
     <title>registrylock_ac88663e:w-&gt;registrylock_ac88663e:e</title> 
-     <path fill="none" stroke="black" d="M4478.18,-3766.88C4416.43,-3841.38 4430.31,-4052.91 4652.5,-4052.91 4879.78,-4052.91 4889.1,-4041.87 4822.46,-4006.57" /> 
-     <polygon fill="black" stroke="black" points="4484.34,-3760.88 4494.64,-3757.13 4487.92,-3757.39 4491.5,-3753.91 4491.5,-3753.91 4491.5,-3753.91 4487.92,-3757.39 4488.36,-3750.68 4484.34,-3760.88 4484.34,-3760.88" /> 
-     <ellipse fill="none" stroke="black" cx="4481.47" cy="-3763.67" rx="4" ry="4" /> 
-     <polygon fill="black" stroke="black" points="4812.08,-4006.8 4816.7,-3997.93 4818.47,-3998.86 4813.85,-4007.73 4812.08,-4006.8" /> 
-     <polyline fill="none" stroke="black" points="4813.5,-4001.91 4817.93,-4004.22 " /> 
-     <polygon fill="black" stroke="black" points="4816.51,-4009.11 4821.13,-4000.24 4822.9,-4001.17 4818.29,-4010.04 4816.51,-4009.11" /> 
-     <polyline fill="none" stroke="black" points="4817.93,-4004.22 4822.37,-4006.52 " /> 
+     <path fill="none" stroke="black" d="M4497.18,-3766.88C4435.35,-3841.38 4448.16,-4052.91 4653,-4052.91 4862.54,-4052.91 4871.13,-4041.87 4804.46,-4006.57" /> 
+     <polygon fill="black" stroke="black" points="4503.33,-3760.88 4513.64,-3757.13 4506.92,-3757.39 4510.5,-3753.91 4510.5,-3753.91 4510.5,-3753.91 4506.92,-3757.39 4507.36,-3750.68 4503.33,-3760.88 4503.33,-3760.88" /> 
+     <ellipse fill="none" stroke="black" cx="4500.47" cy="-3763.67" rx="4" ry="4" /> 
+     <polygon fill="black" stroke="black" points="4794.08,-4006.8 4798.7,-3997.93 4800.47,-3998.86 4795.85,-4007.73 4794.08,-4006.8" /> 
+     <polyline fill="none" stroke="black" points="4795.5,-4001.91 4799.94,-4004.21 " /> 
+     <polygon fill="black" stroke="black" points="4798.51,-4009.11 4803.13,-4000.24 4804.9,-4001.17 4800.29,-4010.04 4798.51,-4009.11" /> 
+     <polyline fill="none" stroke="black" points="4799.94,-4004.21 4804.37,-4006.52 " /> 
     <text text-anchor="start" x="4571" y="-4056.71" font-family="Helvetica,sans-Serif" font-size="14.00">
      fk2lhcwpxlnqijr96irylrh1707
     </text> 
@@ -12706,12 +12706,12 @@ td.section {
    </tr> 
    <tr> 
     <td class="spacer"></td> 
-     <td class="minwidth">lock_completion_timestamp</td> 
+     <td class="minwidth">lock_completion_time</td> 
     <td class="minwidth">timestamptz</td> 
    </tr> 
    <tr> 
     <td class="spacer"></td> 
-     <td class="minwidth">lock_request_timestamp</td> 
+     <td class="minwidth">lock_request_time</td> 
     <td class="minwidth">timestamptz not null</td> 
    </tr> 
    <tr> 
@@ -12746,12 +12746,12 @@ td.section {
    </tr> 
    <tr> 
     <td class="spacer"></td> 
-     <td class="minwidth">unlock_request_timestamp</td> 
+     <td class="minwidth">unlock_request_time</td> 
     <td class="minwidth">timestamptz</td> 
    </tr> 
    <tr> 
     <td class="spacer"></td> 
-     <td class="minwidth">unlock_completion_timestamp</td> 
+     <td class="minwidth">unlock_completion_time</td> 
     <td class="minwidth">timestamptz</td> 
    </tr> 
    <tr> 
@@ -88,3 +88,4 @@ V87__fix_super_domain_fk.sql
 V88__transfer_billing_cancellation_history_id.sql
 V89__host_history_host_deferred.sql
 V90__update_timestamp.sql
+V91__defer_fkeys.sql
@@ -0,0 +1,48 @@
+-- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- We require that *History table writes come after their corresponding
+-- EppResource writes when replaying transactions from Datastore.
+-- The alterations here serve to break cycles necessary to write the
+-- resource first.
+
+ALTER TABLE "BillingEvent" DROP CONSTRAINT fk_billing_event_domain_history;
+ALTER TABLE "BillingEvent" ADD CONSTRAINT fk_billing_event_domain_history
+   FOREIGN KEY (domain_repo_id, domain_history_revision_id)
+   REFERENCES "DomainHistory"(domain_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "BillingRecurrence" DROP CONSTRAINT fk_billing_recurrence_domain_history;
+ALTER TABLE "BillingRecurrence" ADD CONSTRAINT fk_billing_recurrence_domain_history
+   FOREIGN KEY (domain_repo_id, domain_history_revision_id)
+   REFERENCES "DomainHistory"(domain_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "BillingCancellation" DROP CONSTRAINT fk_billing_cancellation_domain_history;
+ALTER TABLE "BillingCancellation" ADD CONSTRAINT fk_billing_cancellation_domain_history
+   FOREIGN KEY (domain_repo_id, domain_history_revision_id)
+   REFERENCES "DomainHistory"(domain_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "PollMessage" DROP CONSTRAINT fk_poll_message_domain_history;
+ALTER TABLE "PollMessage" ADD CONSTRAINT fk_poll_message_domain_history
+   FOREIGN KEY (domain_repo_id, domain_history_revision_id)
+   REFERENCES "DomainHistory"(domain_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "PollMessage" DROP CONSTRAINT fk_poll_message_contact_history;
+ALTER TABLE "PollMessage" ADD CONSTRAINT fk_poll_message_contact_history
+   FOREIGN KEY (contact_repo_id, contact_history_revision_id)
+   REFERENCES "ContactHistory"(contact_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
@@ -1920,7 +1920,7 @@ ALTER TABLE ONLY public."BillingCancellation"
 --

 ALTER TABLE ONLY public."BillingCancellation"
-    ADD CONSTRAINT fk_billing_cancellation_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_billing_cancellation_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;


 --
@@ -1952,7 +1952,7 @@ ALTER TABLE ONLY public."BillingEvent"
 --

 ALTER TABLE ONLY public."BillingEvent"
-    ADD CONSTRAINT fk_billing_event_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_billing_event_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;


 --
@@ -1968,7 +1968,7 @@ ALTER TABLE ONLY public."BillingEvent"
 --

 ALTER TABLE ONLY public."BillingRecurrence"
-    ADD CONSTRAINT fk_billing_recurrence_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_billing_recurrence_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;


 --
@@ -2216,7 +2216,7 @@ ALTER TABLE ONLY public."HostHistory"
 --

 ALTER TABLE ONLY public."PollMessage"
-    ADD CONSTRAINT fk_poll_message_contact_history FOREIGN KEY (contact_repo_id, contact_history_revision_id) REFERENCES public."ContactHistory"(contact_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_poll_message_contact_history FOREIGN KEY (contact_repo_id, contact_history_revision_id) REFERENCES public."ContactHistory"(contact_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;


 --
@@ -2232,7 +2232,7 @@ ALTER TABLE ONLY public."PollMessage"
 --

 ALTER TABLE ONLY public."PollMessage"
-    ADD CONSTRAINT fk_poll_message_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_poll_message_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;


 --