Stop writing ClaimsList to Datastore (#1169)

* Stop writing ClaimsList to Datastore

* Fix some failing tests

* Rename ClaimsListShard to ClaimsList
This commit is contained in:
sarahcaseybot
2021-05-20 15:44:40 -04:00
committed by GitHub
parent 02eb7cfcc3
commit 642405375b
24 changed files with 94 additions and 583 deletions
@@ -55,7 +55,7 @@ import google.registry.model.registrar.RegistrarContact;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.ReservedList;
import google.registry.model.server.Lock;
import google.registry.model.tmch.ClaimsListShard;
import google.registry.model.tmch.ClaimsList;
import google.registry.model.translators.VKeyTranslatorFactory;
import google.registry.persistence.VKey;
import google.registry.persistence.transaction.JpaTransactionManager;
@@ -405,7 +405,7 @@ public class ReplayCommitLogsToSqlActionTest {
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
// Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant)
Key<ClaimsListShard> claimsListKey = Key.create(ClaimsListShard.class, 1L);
Key<ClaimsList> claimsListKey = Key.create(ClaimsList.class, 1L);
saveDiffFile(
gcsService,
createCheckpoint(now.minusMinutes(1)),
@@ -47,8 +47,8 @@ import google.registry.model.host.HostHistory;
import google.registry.model.index.EppResourceIndex;
import google.registry.model.index.EppResourceIndexBucket;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
import google.registry.model.tmch.ClaimsListShard;
import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListDao;
import google.registry.testing.DatabaseHelper;
import google.registry.testing.TaskQueueHelper.TaskMatcher;
import google.registry.util.TypeUtils.TypeInstantiator;
@@ -115,9 +115,9 @@ public abstract class ResourceFlowTestCase<F extends Flow, R extends EppResource
return new TypeInstantiator<R>(getClass()) {}.getExactType();
}
/** Persists a testing claims list to Datastore that contains a single shard. */
/** Persists a testing claims list to Cloud SQL. */
protected void persistClaimsList(ImmutableMap<String, String> labelsToKeys) {
ClaimsListDualDatabaseDao.save(ClaimsListShard.create(clock.nowUtc(), labelsToKeys));
ClaimsListDao.save(ClaimsList.create(clock.nowUtc(), labelsToKeys));
}
@Test
@@ -18,7 +18,6 @@ import static com.google.common.truth.Truth.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.google.common.collect.ImmutableMap;
import com.google.common.truth.Truth8;
import google.registry.persistence.transaction.JpaTestRules;
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
import google.registry.testing.DatastoreEntityExtension;
@@ -28,8 +27,8 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for {@link ClaimsListSqlDao}. */
public class ClaimsListSqlDaoTest {
/** Unit tests for {@link ClaimsListDao}. */
public class ClaimsListDaoTest {
private final FakeClock fakeClock = new FakeClock();
@@ -43,55 +42,51 @@ public class ClaimsListSqlDaoTest {
@Test
void save_insertsClaimsListSuccessfully() {
ClaimsListShard claimsList =
ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsListSqlDao.save(claimsList);
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get();
ClaimsList claimsList =
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsListDao.save(claimsList);
ClaimsList insertedClaimsList = ClaimsListDao.get();
assertClaimsListEquals(claimsList, insertedClaimsList);
assertThat(insertedClaimsList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc());
}
@Test
void save_fail_duplicateId() {
ClaimsListShard claimsList =
ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsListSqlDao.save(claimsList);
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get();
ClaimsList claimsList =
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsListDao.save(claimsList);
ClaimsList insertedClaimsList = ClaimsListDao.get();
assertClaimsListEquals(claimsList, insertedClaimsList);
// Save ClaimsList with existing revisionId should fail because revisionId is the primary key.
assertThrows(PersistenceException.class, () -> ClaimsListSqlDao.save(insertedClaimsList));
assertThrows(PersistenceException.class, () -> ClaimsListDao.save(insertedClaimsList));
}
@Test
void save_claimsListWithNoEntries() {
ClaimsListShard claimsList = ClaimsListShard.create(fakeClock.nowUtc(), ImmutableMap.of());
ClaimsListSqlDao.save(claimsList);
ClaimsListShard insertedClaimsList = ClaimsListSqlDao.get().get();
ClaimsList claimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of());
ClaimsListDao.save(claimsList);
ClaimsList insertedClaimsList = ClaimsListDao.get();
assertClaimsListEquals(claimsList, insertedClaimsList);
assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty();
}
@Test
void getCurrent_returnsEmptyListIfTableIsEmpty() {
Truth8.assertThat(ClaimsListSqlDao.get()).isEmpty();
assertThat(ClaimsListDao.get().labelsToKeys).isEmpty();
}
@Test
void getCurrent_returnsLatestClaims() {
ClaimsListShard oldClaimsList =
ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsListShard newClaimsList =
ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
ClaimsListSqlDao.save(oldClaimsList);
ClaimsListSqlDao.save(newClaimsList);
assertClaimsListEquals(newClaimsList, ClaimsListSqlDao.get().get());
ClaimsList oldClaimsList =
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
ClaimsList newClaimsList =
ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
ClaimsListDao.save(oldClaimsList);
ClaimsListDao.save(newClaimsList);
assertClaimsListEquals(newClaimsList, ClaimsListDao.get());
}
private void assertClaimsListEquals(ClaimsListShard left, ClaimsListShard right) {
private void assertClaimsListEquals(ClaimsList left, ClaimsList right) {
assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId());
assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime());
assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys());
@@ -1,86 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.tmch;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.google.common.collect.ImmutableMap;
import google.registry.config.RegistryEnvironment;
import google.registry.model.EntityTestCase;
import google.registry.testing.SystemPropertyExtension;
import org.joda.time.Duration;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for {@link ClaimsListDualDatabaseDao}. */
public class ClaimsListDualDatabaseDaoTest extends EntityTestCase {
@RegisterExtension
@Order(value = Integer.MAX_VALUE)
final SystemPropertyExtension systemPropertyExtension = new SystemPropertyExtension();
@Test
void testGetList_missingOfy() {
ClaimsListSqlDao.save(createClaimsList());
assertThat(assertThrows(IllegalStateException.class, ClaimsListDualDatabaseDao::get))
.hasMessageThat()
.isEqualTo("Claims list found in Cloud SQL but not in Datastore.");
}
@Test
void testGetList_fromSql_different() {
ClaimsListShard.create(fakeClock.nowUtc(), ImmutableMap.of("foo", "bar")).saveToDatastore();
ClaimsListSqlDao.save(createClaimsList());
assertThat(assertThrows(IllegalStateException.class, ClaimsListDualDatabaseDao::get))
.hasMessageThat()
.isEqualTo(
"Unequal claims lists detected:\n"
+ "Domain label label1 with key key1 only appears in Cloud SQL.\n"
+ "Domain label label2 with key key2 only appears in Cloud SQL.\n"
+ "Domain label foo with key bar only appears in Datastore.\n");
}
@Test
void testSaveAndGet() {
tm().transact(() -> ClaimsListDualDatabaseDao.save(createClaimsList()));
assertAboutImmutableObjects()
.that(ClaimsListDualDatabaseDao.get())
.isEqualExceptFields(createClaimsList(), "id", "revisionId", "creationTimestamp");
}
@Test
void testGet_empty() {
assertThat(tm().transact(ClaimsListDualDatabaseDao::get).getLabelsToKeys()).isEmpty();
}
@Test
void testGetList_missingOfy_notInTest() {
RegistryEnvironment.PRODUCTION.setup(systemPropertyExtension);
fakeClock.advanceBy(Duration.standardDays(5));
ClaimsListSqlDao.save(createClaimsList());
// Shouldn't fail in production
assertThat(ClaimsListDualDatabaseDao.get().getLabelsToKeys())
.isEqualTo(createClaimsList().getLabelsToKeys());
}
private ClaimsListShard createClaimsList() {
return ClaimsListShard.create(
fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
}
}
@@ -1,116 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.tmch;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static org.joda.time.DateTimeZone.UTC;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.google.common.collect.ImmutableMap;
import com.googlecode.objectify.Key;
import google.registry.model.tmch.ClaimsListShard.ClaimsListRevision;
import google.registry.model.tmch.ClaimsListShard.UnshardedSaveException;
import google.registry.testing.AppEngineExtension;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for {@link ClaimsListShard}. */
public class ClaimsListShardTest {
@RegisterExtension
public final AppEngineExtension appEngine =
AppEngineExtension.builder().withDatastoreAndCloudSql().build();
private final int shardSize = 10;
@Test
void test_unshardedSaveFails() {
assertThrows(
UnshardedSaveException.class,
() ->
tm().transact(
() -> {
ClaimsListShard claimsList =
ClaimsListShard.create(
tm().getTransactionTime(), ImmutableMap.of("a", "b"));
claimsList.id = 1; // Without an id this won't save anyways.
claimsList.parent = ClaimsListRevision.createKey();
auditedOfy().saveWithoutBackup().entity(claimsList).now();
}));
}
@Test
void testGet_safelyLoadsEmptyClaimsList_whenNoShardsExist() {
assertThat(ClaimsListShard.getFromDatastore()).isEmpty();
}
@Test
void test_savesAndGets_withSharding() {
// Create a ClaimsList that will need 4 shards to save.
Map<String, String> labelsToKeys = new HashMap<>();
for (int i = 0; i <= shardSize * 3; i++) {
labelsToKeys.put(Integer.toString(i), Integer.toString(i));
}
DateTime now = DateTime.now(UTC);
// Save it with sharding, and make sure that reloading it works.
ClaimsListShard unsharded = ClaimsListShard.create(now, ImmutableMap.copyOf(labelsToKeys));
unsharded.saveToDatastore(shardSize);
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
.isEqualTo(unsharded.labelsToKeys);
List<ClaimsListShard> shards1 = auditedOfy().load().type(ClaimsListShard.class).list();
assertThat(shards1).hasSize(4);
assertThat(ClaimsListShard.getFromDatastore().get().getClaimKey("1")).hasValue("1");
assertThat(ClaimsListShard.getFromDatastore().get().getClaimKey("a")).isEmpty();
assertThat(ClaimsListShard.getCurrentRevision()).isEqualTo(shards1.get(0).parent);
// Create a smaller ClaimsList that will need only 2 shards to save.
labelsToKeys = new HashMap<>();
for (int i = 0; i <= shardSize; i++) {
labelsToKeys.put(Integer.toString(i), Integer.toString(i));
}
unsharded = ClaimsListShard.create(now.plusDays(1), ImmutableMap.copyOf(labelsToKeys));
unsharded.saveToDatastore(shardSize);
auditedOfy().clearSessionCache();
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
.hasSize(unsharded.labelsToKeys.size());
assertThat(ClaimsListShard.getFromDatastore().get().labelsToKeys)
.isEqualTo(unsharded.labelsToKeys);
List<ClaimsListShard> shards2 = auditedOfy().load().type(ClaimsListShard.class).list();
assertThat(shards2).hasSize(2);
// Expect that the old revision is deleted.
assertThat(ClaimsListShard.getCurrentRevision()).isEqualTo(shards2.get(0).parent);
}
/**
* Returns a created claims list shard with the specified parent key for testing purposes only.
*/
public static ClaimsListShard createTestClaimsListShard(
DateTime creationTime,
ImmutableMap<String, String> labelsToKeys,
Key<ClaimsListRevision> revision) {
ClaimsListShard claimsList = ClaimsListShard.create(creationTime, labelsToKeys);
claimsList.isShard = true;
claimsList.parent = revision;
return claimsList;
}
}
@@ -34,7 +34,7 @@ import google.registry.model.server.KmsSecretRevisionSqlDaoTest;
import google.registry.model.server.LockTest;
import google.registry.model.server.ServerSecretTest;
import google.registry.model.smd.SignedMarkRevocationListDaoTest;
import google.registry.model.tmch.ClaimsListSqlDaoTest;
import google.registry.model.tmch.ClaimsListDaoTest;
import google.registry.model.tmch.TmchCrlTest;
import google.registry.persistence.transaction.JpaEntityCoverageExtension;
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
@@ -82,7 +82,7 @@ import org.junit.runner.RunWith;
BeforeSuiteTest.class,
AllocationTokenTest.class,
BillingEventTest.class,
ClaimsListSqlDaoTest.class,
ClaimsListDaoTest.class,
ContactHistoryTest.class,
ContactResourceTest.class,
CursorTest.class,
@@ -20,8 +20,8 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
import google.registry.model.tmch.ClaimsListShard;
import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListDao;
import java.util.Optional;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
@@ -38,7 +38,7 @@ class TmchDnlActionTest extends TmchActionTestCase {
@Test
void testDnl() throws Exception {
assertThat(ClaimsListDualDatabaseDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
when(httpResponse.getContent())
.thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read())
.thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read());
@@ -50,7 +50,7 @@ class TmchDnlActionTest extends TmchActionTestCase {
.isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig");
// Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
ClaimsListShard claimsList = ClaimsListDualDatabaseDao.get();
ClaimsList claimsList = ClaimsListDao.get();
assertThat(claimsList.getTmdbGenerationTime())
.isEqualTo(DateTime.parse("2013-11-24T23:15:37.4Z"));
assertThat(claimsList.getClaimKey("xn----7sbejwbn3axu3d"))
@@ -20,8 +20,8 @@ import static java.nio.file.Files.readAllLines;
import static org.joda.time.DateTimeZone.UTC;
import com.google.common.collect.ImmutableMap;
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
import google.registry.model.tmch.ClaimsListShard;
import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListDao;
import java.io.File;
import java.nio.file.Files;
import org.joda.time.DateTime;
@@ -32,8 +32,7 @@ class GetClaimsListCommandTest extends CommandTestCase<GetClaimsListCommand> {
@Test
void testSuccess_getWorks() throws Exception {
ClaimsListDualDatabaseDao.save(
ClaimsListShard.create(DateTime.now(UTC), ImmutableMap.of("a", "1", "b", "2")));
ClaimsListDao.save(ClaimsList.create(DateTime.now(UTC), ImmutableMap.of("a", "1", "b", "2")));
File output = tmpDir.resolve("claims.txt").toFile();
runCommand("--output=" + output.getAbsolutePath());
assertThat(readAllLines(output.toPath(), UTF_8)).containsExactly("a,1", "b,2");
@@ -41,8 +40,7 @@ class GetClaimsListCommandTest extends CommandTestCase<GetClaimsListCommand> {
@Test
void testSuccess_endsWithNewline() throws Exception {
ClaimsListDualDatabaseDao.save(
ClaimsListShard.create(DateTime.now(UTC), ImmutableMap.of("a", "1")));
ClaimsListDao.save(ClaimsList.create(DateTime.now(UTC), ImmutableMap.of("a", "1")));
File output = tmpDir.resolve("claims.txt").toFile();
runCommand("--output=" + output.getAbsolutePath());
assertThat(new String(Files.readAllBytes(output.toPath()), UTF_8)).endsWith("\n");
@@ -18,8 +18,8 @@ import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
import google.registry.model.tmch.ClaimsListShard;
import google.registry.model.tmch.ClaimsList;
import google.registry.model.tmch.ClaimsListDao;
import java.io.FileNotFoundException;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
@@ -37,7 +37,7 @@ class UploadClaimsListCommandTest extends CommandTestCase<UploadClaimsListComman
"anotherexample,2013041500/A/C/7/rHdC4wnrWRvPY6nneCVtQhFj0000000003,2011-08-16T12:00:00.0Z");
runCommand("--force", filename);
ClaimsListShard claimsList = ClaimsListDualDatabaseDao.get();
ClaimsList claimsList = ClaimsListDao.get();
assertThat(claimsList.getTmdbGenerationTime())
.isEqualTo(DateTime.parse("2012-08-16T00:00:00.0Z"));
assertThat(claimsList.getClaimKey("example"))
@@ -1,4 +1,4 @@
ClaimsListShard
ClaimsList
ClaimsListSingleton
Cursor
DatabaseTransitionSchedule
@@ -847,20 +847,20 @@ class google.registry.model.server.ServerSecret {
long leastSignificant;
long mostSignificant;
}
class google.registry.model.tmch.ClaimsListShard {
class google.registry.model.tmch.ClaimsList {
@Id long id;
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListRevision> parent;
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListRevision> parent;
java.util.Map<java.lang.String, java.lang.String> labelsToKeys;
org.joda.time.DateTime creationTime;
}
class google.registry.model.tmch.ClaimsListShard$ClaimsListRevision {
class google.registry.model.tmch.ClaimsList$ClaimsListRevision {
@Id long versionId;
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListSingleton> parent;
@Parent com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListSingleton> parent;
}
class google.registry.model.tmch.ClaimsListShard$ClaimsListSingleton {
class google.registry.model.tmch.ClaimsList$ClaimsListSingleton {
@Id long id;
@Parent com.googlecode.objectify.Key<google.registry.model.common.EntityGroupRoot> parent;
com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsListShard$ClaimsListRevision> activeRevision;
com.googlecode.objectify.Key<google.registry.model.tmch.ClaimsList$ClaimsListRevision> activeRevision;
}
class google.registry.model.tmch.TmchCrl {
@Id long id;