mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-09-02 22:27:07 +00:00
Code Issues Releases Wiki Activity

Fix runtime issues with commit-log-to-SQL replay (#1240)

Browse Source 
* Fix runtime issues with commit-log-to-SQL replay

- We now use a more intelligent prefix to narrow the listObjects search
space in GCS. Otherwise, we're returning >30k objects which can take
roughly 50 seconds. This results in a listObjects time of 1-3 seconds.

- We now search hour by hour to efficiently make use of the prefixing.
Basically, we keep searching for new files until we hit the current time
or until we hit the overall replay timeout.

- Dry-run only prints out the first hour's worth of files
This commit is contained in:
gbrodman
2021-07-22 13:59:28 -04:00
committed by GitHub
parent 3beb207fcc
commit 38c8e81690
5 changed files with 146 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java
+19 -7
View File
@@ -18,6 +18,7 @@ import static com.google.common.collect.Iterables.transform;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
import static google.registry.backup.GcsDiffFileLister.getCommitLogDiffPrefix;
import static org.joda.time.DateTimeZone.UTC;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.fail;
@@ -61,7 +62,7 @@ public class GcsDiffFileListerTest {
@BeforeEach
void beforeEach() throws Exception {
diffLister.gcsUtils = gcsUtils;
diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
diffLister.executorProvider = MoreExecutors::newDirectExecutorService;
diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
for (int i = 0; i < 5; i++) {
addGcsFile(i, i + 1);
@@ -189,12 +190,23 @@ public class GcsDiffFileListerTest {
@Test
void testList_toTimeSpecified() {
assertThat(listDiffFiles(
now.minusMinutes(4).minusSeconds(1), now.minusMinutes(2).plusSeconds(1)))
.containsExactly(
now.minusMinutes(4),
now.minusMinutes(3),
now.minusMinutes(2))
assertThat(
listDiffFiles(now.minusMinutes(4).minusSeconds(1), now.minusMinutes(2).plusSeconds(1)))
.containsExactly(now.minusMinutes(4), now.minusMinutes(3), now.minusMinutes(2))
.inOrder();
}
@Test
void testPrefix_lengthened() {
DateTime from = DateTime.parse("2021-05-11T06:48:00.070Z");
assertThat(getCommitLogDiffPrefix(from, null)).isEqualTo("commit_diff_until_");
assertThat(getCommitLogDiffPrefix(from, DateTime.parse("2021-07-01")))
.isEqualTo("commit_diff_until_2021-");
assertThat(getCommitLogDiffPrefix(from, DateTime.parse("2021-05-21")))
.isEqualTo("commit_diff_until_2021-05-");
assertThat(getCommitLogDiffPrefix(from, DateTime.parse("2021-05-11T09:48:00.070Z")))
.isEqualTo("commit_diff_until_2021-05-11T");
assertThat(getCommitLogDiffPrefix(from, DateTime.parse("2021-05-11T06:59:00.070Z")))
.isEqualTo("commit_diff_until_2021-05-11T06:");
}
}
core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java
+17 -14
View File
@@ -132,7 +132,7 @@ public class ReplayCommitLogsToSqlActionTest {
action.gcsBucket = "gcs bucket";
action.diffLister = new GcsDiffFileLister();
action.diffLister.gcsUtils = gcsUtils;
action.diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
action.diffLister.executorProvider = MoreExecutors::newDirectExecutorService;
action.diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
ofyTm()
.transact(
@@ -200,7 +200,7 @@ public class ReplayCommitLogsToSqlActionTest {
CommitLogMutation.create(manifest2Key, TestObject.create("f")));
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
fakeClock.advanceOneMilli();
runAndAssertSuccess(now);
runAndAssertSuccess(now, 2);
assertExpectedIds("previous to keep", "b", "d", "e", "f");
}
@@ -211,7 +211,7 @@ public class ReplayCommitLogsToSqlActionTest {
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
saveDiffFile(gcsUtils, createCheckpoint(now.minusMillis(2)));
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMillis(1)));
runAndAssertSuccess(now.minusMillis(1));
runAndAssertSuccess(now.minusMillis(1), 0);
assertExpectedIds("previous to keep");
}
@@ -235,8 +235,10 @@ public class ReplayCommitLogsToSqlActionTest {
assertThat(response.getStatus()).isEqualTo(SC_OK);
assertThat(response.getPayload())
.isEqualTo(
"Running in dry-run mode; would have processed %d files. They are (limit 10):\n"
+ "commit_diff_until_1999-12-31T23:59:00.000Z");
"Running in dry-run mode, the first set of commit log files processed would be from "
+ "searching from 1999-12-31T23:59:00.000Z to 1999-12-31T23:59:59.999Z and would "
+ "contain 1 file(s). They are (limit 10): \n"
+ "[commit_diff_until_1999-12-31T23:59:00.000Z]");
}
@Test
@@ -253,7 +255,7 @@ public class ReplayCommitLogsToSqlActionTest {
CommitLogManifest.create(bucketKey, now, null),
CommitLogMutation.create(manifestKey, TestObject.create("a")),
CommitLogMutation.create(manifestKey, TestObject.create("b")));
runAndAssertSuccess(now.minusMinutes(1));
runAndAssertSuccess(now.minusMinutes(1), 1);
assertExpectedIds("previous to keep", "a", "b");
}
@@ -275,7 +277,7 @@ public class ReplayCommitLogsToSqlActionTest {
getBucketKey(1),
now,
ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
runAndAssertSuccess(now.minusMinutes(1));
runAndAssertSuccess(now.minusMinutes(1), 1);
assertExpectedIds("previous to keep");
}
@@ -347,7 +349,7 @@ public class ReplayCommitLogsToSqlActionTest {
domainMutation,
contactMutation);
runAndAssertSuccess(now.minusMinutes(1));
runAndAssertSuccess(now.minusMinutes(1), 1);
// Verify two things:
// 1. that the contact insert occurred before the domain insert (necessary for FK ordering)
// even though the domain came first in the file
@@ -390,7 +392,7 @@ public class ReplayCommitLogsToSqlActionTest {
CommitLogManifest.create(
getBucketKey(1), now.minusMinutes(1).plusMillis(1), ImmutableSet.of()),
contactMutation);
runAndAssertSuccess(now.minusMinutes(1).plusMillis(1));
runAndAssertSuccess(now.minusMinutes(1).plusMillis(1), 1);
// Verify that the delete occurred first (because it was in the first transaction) even though
// deletes have higher weight
ArgumentCaptor<Object> putCaptor = ArgumentCaptor.forClass(Object.class);
@@ -435,7 +437,7 @@ public class ReplayCommitLogsToSqlActionTest {
throw new RuntimeException(e);
}
});
runAndAssertSuccess(now.minusMinutes(1));
runAndAssertSuccess(now.minusMinutes(1), 1);
// jpaTm()::put should only have been called with the checkpoint
verify(spy, times(2)).put(any(SqlReplayCheckpoint.class));
verify(spy, times(2)).put(any());
@@ -460,7 +462,7 @@ public class ReplayCommitLogsToSqlActionTest {
// one object only exists in Datastore, one is dually-written (so isn't replicated)
ImmutableSet.of(getCrossTldKey(), claimsListKey)));
runAndAssertSuccess(now.minusMinutes(1));
runAndAssertSuccess(now.minusMinutes(1), 1);
verify(spy, times(0)).delete(any(VKey.class));
}
@@ -503,7 +505,7 @@ public class ReplayCommitLogsToSqlActionTest {
createCheckpoint(now.minusMinutes(1)),
CommitLogManifest.create(bucketKey, now, null),
CommitLogMutation.create(manifestKey, TestObject.create("a")));
runAndAssertSuccess(now.minusMinutes(1));
runAndAssertSuccess(now.minusMinutes(1), 1);
assertThat(TestObject.beforeSqlSaveCallCount).isEqualTo(1);
}
@@ -522,11 +524,12 @@ public class ReplayCommitLogsToSqlActionTest {
assertThat(TestObject.beforeSqlDeleteCallCount).isEqualTo(1);
}
private void runAndAssertSuccess(DateTime expectedCheckpointTime) {
private void runAndAssertSuccess(DateTime expectedCheckpointTime, int numFiles) {
action.run();
assertThat(response.getStatus()).isEqualTo(SC_OK);
assertThat(response.getPayload())
.isEqualTo("ReplayCommitLogsToSqlAction completed successfully.");
.isEqualTo(
String.format("Caught up to current time after replaying %d file(s).", numFiles));
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(expectedCheckpointTime);
}
core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java
+1 -1
View File
@@ -88,7 +88,7 @@ public class RestoreCommitLogsActionTest {
action.gcsBucketOverride = Optional.empty();
action.diffLister = new GcsDiffFileLister();
action.diffLister.gcsUtils = gcsUtils;
action.diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
action.diffLister.executorProvider = MoreExecutors::newDirectExecutorService;
action.diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
}