mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-02-09 14:30:33 +00:00
Code Issues Releases Wiki Activity

Write commit logs during SQL->DS replay (#1438)

Browse Source 
* Write commit logs during SQL->DS replay

Previously, we had no way to ignore read-only mode while still writing
commit log backups. Now, we added this so we can write commit logs in
the SQL->DS replay.

Note:
- When moving to either of the DATASTORE_PRIMARY stages, one must
manually set the SqlReplayCheckpoint first. We don't write to SQL with
backup in this stage because we already wrote the transaction in
question to Datastore. The fact that we manually set the replay
checkpoint means that we'll ignore the extra commit logs that might
otherwise cause problems if we switched back and forth from
DATASTORE_PRIMARY to SQL_PRIMARY.

- The commit logs written during the SQL_PRIMARY phase will, ideally, be
unused. We write them here only so that in the event of a rollback to
Datastore, we will have them for RDE purposes.
This commit is contained in:
gbrodman
2021-12-01 11:31:03 -05:00
committed by GitHub
parent f054bb2694
commit 207fc49d64
16 changed files with 106 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java
View File

@@ -382,10 +382,10 @@ public class ReplayCommitLogsToSqlActionTest {
// even though the domain came first in the file
// 2. that the allocation token delete occurred after the insertions
InOrder inOrder = Mockito.inOrder(spy);
inOrder.verify(spy).putIgnoringReadOnly(any(ContactResource.class));
inOrder.verify(spy).putIgnoringReadOnly(any(DomainBase.class));
inOrder.verify(spy).deleteIgnoringReadOnly(toDelete.createVKey());
inOrder.verify(spy).putIgnoringReadOnly(any(SqlReplayCheckpoint.class));
inOrder.verify(spy).putIgnoringReadOnlyWithoutBackup(any(ContactResource.class));
inOrder.verify(spy).putIgnoringReadOnlyWithoutBackup(any(DomainBase.class));
inOrder.verify(spy).deleteIgnoringReadOnlyWithoutBackup(toDelete.createVKey());
inOrder.verify(spy).putIgnoringReadOnlyWithoutBackup(any(SqlReplayCheckpoint.class));
}
@Test
@@ -424,8 +424,8 @@ public class ReplayCommitLogsToSqlActionTest {
// deletes have higher weight
ArgumentCaptor<Object> putCaptor = ArgumentCaptor.forClass(Object.class);
InOrder inOrder = Mockito.inOrder(spy);
inOrder.verify(spy).deleteIgnoringReadOnly(contact.createVKey());
inOrder.verify(spy).putIgnoringReadOnly(putCaptor.capture());
inOrder.verify(spy).deleteIgnoringReadOnlyWithoutBackup(contact.createVKey());
inOrder.verify(spy).putIgnoringReadOnlyWithoutBackup(putCaptor.capture());
assertThat(putCaptor.getValue().getClass()).isEqualTo(ContactResource.class);
assertThat(jpaTm().transact(() -> jpaTm().loadByKey(contact.createVKey()).getEmailAddress()))
.isEqualTo("replay@example.tld");
@@ -467,9 +467,9 @@ public class ReplayCommitLogsToSqlActionTest {
});
runAndAssertSuccess(now.minusMinutes(1), 1, 1);
// jpaTm()::putIgnoringReadOnly should only have been called with the checkpoint and the lock
verify(spy, times(2)).putIgnoringReadOnly(any(SqlReplayCheckpoint.class));
verify(spy).putIgnoringReadOnly(any(Lock.class));
verify(spy, times(3)).putIgnoringReadOnly(any());
verify(spy, times(2)).putIgnoringReadOnlyWithoutBackup(any(SqlReplayCheckpoint.class));
verify(spy).putIgnoringReadOnlyWithoutBackup(any(Lock.class));
verify(spy, times(3)).putIgnoringReadOnlyWithoutBackup(any());
}
@Test

19
core/src/test/java/google/registry/persistence/transaction/TransactionTest.java
View File

@@ -14,7 +14,9 @@
package google.registry.persistence.transaction;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -23,6 +25,8 @@ import com.googlecode.objectify.Key;
import com.googlecode.objectify.annotation.Entity;
import com.googlecode.objectify.annotation.Id;
import google.registry.model.ImmutableObject;
import google.registry.model.ofy.CommitLogManifest;
import google.registry.model.ofy.CommitLogMutation;
import google.registry.model.ofy.Ofy;
import google.registry.persistence.VKey;
import google.registry.testing.AppEngineExtension;
@@ -33,6 +37,8 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.StreamCorruptedException;
import java.util.Comparator;
import java.util.List;
import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -41,7 +47,8 @@ import org.junit.jupiter.api.extension.RegisterExtension;
class TransactionTest {
private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ"));
private final FakeClock fakeClock =
new FakeClock(DateTime.parse("2000-01-01TZ")).setAutoIncrementByOneMilli();
@RegisterExtension
final AppEngineExtension appEngine =
@@ -83,6 +90,16 @@ class TransactionTest {
txn = new Transaction.Builder().addDelete(barEntity.key()).build();
txn.writeToDatastore();
assertThat(ofyTm().exists(barEntity.key())).isEqualTo(false);
assertThat(
auditedOfy().load().type(CommitLogMutation.class).list().stream()
.map(clm -> auditedOfy().load().<TestEntity>fromEntity(clm.getEntity()))
.collect(toImmutableSet()))
.containsExactly(fooEntity, barEntity);
List<CommitLogManifest> manifests = auditedOfy().load().type(CommitLogManifest.class).list();
manifests.sort(Comparator.comparing(CommitLogManifest::getCommitTime));
assertThat(manifests.get(0).getDeletions()).isEmpty();
assertThat(manifests.get(1).getDeletions()).containsExactly(Key.create(barEntity));
}
@Test

2
core/src/test/java/google/registry/testing/DatabaseHelper.java
View File

@@ -1488,7 +1488,7 @@ public class DatabaseHelper {
.transact(
() ->
jpaTm()
.putIgnoringReadOnly(
.putIgnoringReadOnlyWithoutBackup(
new DatabaseMigrationStateSchedule(
DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP)));
DatabaseMigrationStateSchedule.CACHE.invalidateAll();