Create a mechanism for storing / using locks explicitly only in SQL (#1392)

This is used for the replay locks so that Beam pipelines (which will be
used for database comparison) can acquire / release locks as necessary
to avoid database contention. If we're comparing contents of Datastore
and SQL databases, we shouldn't have replay actively running during the
comparison, so the pipeline will grab the locks.

Beam doesn't always play nicely with loading from / saving to Datastore,
so we need to make sure that we store the replay locks in SQL at all
times, even when Datastore is the primary DB.

core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java

@@ -34,6 +34,7 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -111,6 +112,7 @@ public class ReplayCommitLogsToSqlActionTest {
               DelegationSignerData.class,
               DomainBase.class,
               GracePeriod.class,
+              Lock.class,
               PremiumList.class,
               PremiumEntry.class,
               RegistrarContact.class,
@@ -135,6 +137,7 @@ public class ReplayCommitLogsToSqlActionTest {
   @BeforeEach
   void beforeEach() {
     inject.setStaticField(Ofy.class, "clock", fakeClock);
+    lenient().when(requestStatusChecker.getLogId()).thenReturn("requestLogId");
     action.gcsUtils = gcsUtils;
     action.response = response;
     action.requestStatusChecker = requestStatusChecker;
@@ -464,9 +467,10 @@ public class ReplayCommitLogsToSqlActionTest {
               }
             });
     runAndAssertSuccess(now.minusMinutes(1), 1, 1);
-    // jpaTm()::putIgnoringReadOnly should only have been called with the checkpoint
+    // jpaTm()::putIgnoringReadOnly should only have been called with the checkpoint and the lock
     verify(spy, times(2)).putIgnoringReadOnly(any(SqlReplayCheckpoint.class));
-    verify(spy, times(2)).putIgnoringReadOnly(any());
+    verify(spy).putIgnoringReadOnly(any(Lock.class));
+    verify(spy, times(3)).putIgnoringReadOnly(any());
   }
 
   @Test
@@ -506,7 +510,7 @@ public class ReplayCommitLogsToSqlActionTest {
   @Test
   void testFailure_cannotAcquireLock() {
     Truth8.assertThat(
-            Lock.acquire(
+            Lock.acquireSql(
                 ReplayCommitLogsToSqlAction.class.getSimpleName(),
                 null,
                 Duration.standardHours(1),

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -294,7 +294,7 @@ public class ReplicateToDatastoreActionTest {
     RequestStatusChecker requestStatusChecker = mock(RequestStatusChecker.class);
     when(requestStatusChecker.getLogId()).thenReturn("logId");
     Truth8.assertThat(
-            Lock.acquire(
+            Lock.acquireSql(
                 ReplicateToDatastoreAction.class.getSimpleName(),
                 null,
                 Duration.standardHours(1),

core/src/test/java/google/registry/model/server/LockTest.java

@@ -20,6 +20,7 @@ import static google.registry.model.server.Lock.LockState.FREE;
 import static google.registry.model.server.Lock.LockState.IN_USE;
 import static google.registry.model.server.Lock.LockState.OWNER_DIED;
 import static google.registry.model.server.Lock.LockState.TIMED_OUT;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -28,8 +29,10 @@ import static org.mockito.Mockito.when;
 
 import google.registry.model.EntityTestCase;
 import google.registry.model.server.Lock.LockState;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import google.registry.util.RequestStatusChecker;
 import java.util.Optional;
 import org.joda.time.Duration;
@@ -132,6 +135,19 @@ public class LockTest extends EntityTestCase {
     assertThat(acquire("b", ONE_DAY, IN_USE)).isEmpty();
   }
 
+  @TestOfyOnly
+  void testSqlLock_inOfyMode() {
+    Lock.lockMetrics = origLockMetrics;
+    Optional<Lock> lock = Lock.acquireSql(RESOURCE_NAME, null, ONE_DAY, requestStatusChecker, true);
+    assertThat(lock).isPresent();
+    assertThat(DatabaseHelper.loadAllOf(Lock.class)).isEmpty();
+    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Lock.class))).containsExactly(lock.get());
+
+    lock.get().releaseSql();
+    assertThat(DatabaseHelper.loadAllOf(Lock.class)).isEmpty();
+    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Lock.class))).isEmpty();
+  }
+
   @TestOfyAndSql
   void testFailure_emptyResourceName() {
     IllegalArgumentException thrown =

core/src/test/java/google/registry/request/lock/LockHandlerImplTest.java

@@ -40,7 +40,9 @@ final class LockHandlerImplTest {
 
   private final FakeClock clock = new FakeClock(DateTime.parse("2001-08-29T12:20:00Z"));
 
-  @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().build();
+  @RegisterExtension
+  final AppEngineExtension appEngine =
+      AppEngineExtension.builder().withDatastoreAndCloudSql().build();
 
   private static class CountingCallable implements Callable<Void> {
     int numCalled = 0;
@@ -69,18 +71,13 @@ final class LockHandlerImplTest {
   }
 
   private boolean executeWithLocks(Callable<Void> callable, final @Nullable Lock acquiredLock) {
-    LockHandlerImpl lockHandler = new LockHandlerImpl(new RequestStatusCheckerImpl(), clock) {
-      private static final long serialVersionUID = 0L;
-      @Override
-      Optional<Lock> acquire(String resourceName, String tld, Duration leaseLength) {
-        assertThat(resourceName).isEqualTo("resourceName");
-        assertThat(tld).isEqualTo("tld");
-        assertThat(leaseLength).isEqualTo(ONE_DAY);
-        return Optional.ofNullable(acquiredLock);
-      }
-    };
+    return createTestLockHandler(acquiredLock)
+        .executeWithLocks(callable, "tld", ONE_DAY, "resourceName");
+  }
 
-    return lockHandler.executeWithLocks(callable, "tld", ONE_DAY, "resourceName");
+  private boolean executeWithSqlLocks(Callable<Void> callable, final @Nullable Lock acquiredLock) {
+    return createTestLockHandler(acquiredLock)
+        .executeWithSqlLocks(callable, "tld", ONE_DAY, "resourceName");
   }
 
   @Test
@@ -92,6 +89,15 @@ final class LockHandlerImplTest {
     verify(lock, times(1)).release();
   }
 
+  @Test
+  void testSqlLockSucceeds() {
+    Lock lock = mock(Lock.class);
+    CountingCallable countingCallable = new CountingCallable();
+    assertThat(executeWithSqlLocks(countingCallable, lock)).isTrue();
+    assertThat(countingCallable.numCalled).isEqualTo(1);
+    verify(lock, times(1)).release();
+  }
+
   @Test
   void testLockSucceeds_uncheckedException() {
     Lock lock = mock(Lock.class);
@@ -140,4 +146,23 @@ final class LockHandlerImplTest {
     assertThat(executeWithLocks(countingCallable, lock)).isFalse();
     assertThat(countingCallable.numCalled).isEqualTo(0);
   }
+
+  private LockHandler createTestLockHandler(@Nullable Lock acquiredLock) {
+    return new LockHandlerImpl(new RequestStatusCheckerImpl(), clock) {
+      private static final long serialVersionUID = 0L;
+
+      @Override
+      Optional<Lock> acquire(String resourceName, String tld, Duration leaseLength) {
+        assertThat(resourceName).isEqualTo("resourceName");
+        assertThat(tld).isEqualTo("tld");
+        assertThat(leaseLength).isEqualTo(ONE_DAY);
+        return Optional.ofNullable(acquiredLock);
+      }
+
+      @Override
+      Optional<Lock> acquireSql(String resourceName, String tld, Duration leaseLength) {
+        return acquire(resourceName, tld, leaseLength);
+      }
+    };
+  }
 }

core/src/test/java/google/registry/testing/FakeLockHandler.java

@@ -26,11 +26,11 @@ public class FakeLockHandler implements LockHandler {
 
   private static final long serialVersionUID = 6437880915118738492L;
 
-  boolean lockSucceeds;
+  private final boolean lockSucceeds;
 
   /**
    * @param lockSucceeds if true - the lock acquisition will succeed and the callable will be
-   * called. If false, lock acquisition will fail and the caller isn't called.
+   *     called. If false, lock acquisition will fail and the caller isn't called.
    */
   public FakeLockHandler(boolean lockSucceeds) {
     this.lockSucceeds = lockSucceeds;
@@ -38,10 +38,17 @@ public class FakeLockHandler implements LockHandler {
 
   @Override
   public boolean executeWithLocks(
-      final Callable<Void> callable,
-      @Nullable String tld,
-      Duration leaseLength,
-      String... lockNames) {
+      Callable<Void> callable, @Nullable String tld, Duration leaseLength, String... lockNames) {
+    return execute(callable);
+  }
+
+  @Override
+  public boolean executeWithSqlLocks(
+      Callable<Void> callable, @Nullable String tld, Duration leaseLength, String... lockNames) {
+    return execute(callable);
+  }
+
+  private boolean execute(Callable<Void> callable) {
     if (!lockSucceeds) {
       return false;
     }