mirror of
https://github.com/google/nomulus
synced 2026-09-12 11:06:29 +00:00
Stop writing ClaimsList to Datastore (#1169)
* Stop writing ClaimsList to Datastore * Fix some failing tests * Rename ClaimsListShard to ClaimsList
This commit is contained in:
sarahcaseybot
@@ -45,7 +45,7 @@ import google.registry.model.eppinput.ResourceCommand;
|
||||
import google.registry.model.eppoutput.EppResponse;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.HashSet;
|
||||
import java.util.Optional;
|
||||
@@ -104,8 +104,7 @@ public final class DomainClaimsCheckFlow implements Flow {
|
||||
verifyClaimsPeriodNotEnded(registry, now);
|
||||
}
|
||||
}
|
||||
Optional<String> claimKey =
|
||||
ClaimsListDualDatabaseDao.get().getClaimKey(parsedDomain.parts().get(0));
|
||||
Optional<String> claimKey = ClaimsListDao.get().getClaimKey(parsedDomain.parts().get(0));
|
||||
launchChecksBuilder.add(
|
||||
LaunchCheck.create(
|
||||
LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null)));
|
||||
|
||||
@@ -126,7 +126,7 @@ import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.reporting.DomainTransactionRecord;
|
||||
import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.tldconfig.idn.IdnLabelValidator;
|
||||
import google.registry.util.Idn;
|
||||
@@ -990,8 +990,7 @@ public class DomainFlowUtils {
|
||||
static void verifyClaimsNoticeIfAndOnlyIfNeeded(
|
||||
InternetDomainName domainName, boolean hasSignedMarks, boolean hasClaimsNotice)
|
||||
throws EppException {
|
||||
boolean isInClaimsList =
|
||||
ClaimsListDualDatabaseDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
|
||||
boolean isInClaimsList = ClaimsListDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
|
||||
if (hasClaimsNotice && !isInClaimsList) {
|
||||
throw new UnexpectedClaimsNoticeException(domainName.toString());
|
||||
}
|
||||
|
||||
@@ -47,9 +47,9 @@ import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.server.KmsSecretRevision;
|
||||
import google.registry.model.server.Lock;
|
||||
import google.registry.model.server.ServerSecret;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListRevision;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListRevision;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
|
||||
import google.registry.model.tmch.TmchCrl;
|
||||
import google.registry.schema.replay.LastSqlTransaction;
|
||||
|
||||
@@ -64,7 +64,7 @@ public final class EntityClasses {
|
||||
BillingEvent.Modification.class,
|
||||
BillingEvent.OneTime.class,
|
||||
BillingEvent.Recurring.class,
|
||||
ClaimsListShard.class,
|
||||
ClaimsList.class,
|
||||
ClaimsListRevision.class,
|
||||
ClaimsListSingleton.class,
|
||||
CommitLogBucket.class,
|
||||
|
||||
@@ -38,7 +38,7 @@ import google.registry.model.common.TimedTransitionProperty.TimedTransition;
|
||||
import google.registry.model.registry.label.PremiumList;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.smd.SignedMarkRevocationList;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
import java.util.Optional;
|
||||
@@ -61,7 +61,7 @@ public class DatabaseTransitionSchedule extends ImmutableObject implements Datas
|
||||
|
||||
/** The id of the transition schedule. */
|
||||
public enum TransitionId {
|
||||
/** The schedule for migration of {@link ClaimsListShard} entities. */
|
||||
/** The schedule for migration of {@link ClaimsList} entities. */
|
||||
CLAIMS_LIST,
|
||||
/** The schedule for the migration of {@link PremiumList} and {@link ReservedList}. */
|
||||
DOMAIN_LABEL_LISTS,
|
||||
|
||||
+5
-157
@@ -16,22 +16,16 @@ package google.registry.model.tmch;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Throwables.throwIfUnchecked;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.ofy.ObjectifyService.allocateId;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.EmbedMap;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Ignore;
|
||||
import com.googlecode.objectify.annotation.OnSave;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.CreateAutoTimestamp;
|
||||
import google.registry.model.ImmutableObject;
|
||||
@@ -42,12 +36,6 @@ import google.registry.model.annotations.VirtualEntity;
|
||||
import google.registry.model.common.CrossTldSingleton;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
import google.registry.schema.replay.NonReplicatedEntity;
|
||||
import google.registry.util.CollectionUtils;
|
||||
import google.registry.util.Concurrent;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.util.SystemSleeper;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import javax.annotation.Nullable;
|
||||
@@ -65,36 +53,20 @@ import org.joda.time.DateTime;
|
||||
/**
|
||||
* A list of TMCH claims labels and their associated claims keys.
|
||||
*
|
||||
* <p>The claims list is actually sharded into multiple {@link ClaimsListShard} entities to work
|
||||
* around the Datastore limitation of 1M max size per entity. However, when calling {@link
|
||||
* #getFromDatastore} all of the shards are recombined into one {@link ClaimsListShard} object.
|
||||
*
|
||||
* <p>ClaimsList shards are tied to a specific revision and are persisted individually, then the
|
||||
* entire claims list is atomically shifted over to using the new shards by persisting the new
|
||||
* revision object and updating the {@link ClaimsListSingleton} pointing to it. This bypasses the
|
||||
* 10MB per transaction limit.
|
||||
*
|
||||
* <p>Therefore, it is never OK to save an instance of this class directly to Datastore. Instead you
|
||||
* must use the {@link #saveToDatastore} method to do it for you.
|
||||
*
|
||||
* <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
|
||||
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly
|
||||
* succeeds, we will end up with having two exact same claims list with only different {@link
|
||||
* #revisionId}. However, this is not an actual problem because we only use the claims list with
|
||||
* highest {@link #revisionId}.
|
||||
*
|
||||
* <p>TODO(b/162007765): Rename the class to ClaimsList and remove Datastore related fields and
|
||||
* methods.
|
||||
* <p>TODO(b/162007765): Remove Datastore related fields and methods.
|
||||
*/
|
||||
@Entity
|
||||
@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
|
||||
@javax.persistence.Entity(name = "ClaimsList")
|
||||
@Table
|
||||
@InCrossTld
|
||||
public class ClaimsListShard extends ImmutableObject implements NonReplicatedEntity {
|
||||
|
||||
/** The number of claims list entries to store per shard. */
|
||||
private static final int SHARD_SIZE = 10000;
|
||||
public class ClaimsList extends ImmutableObject implements NonReplicatedEntity {
|
||||
|
||||
@Transient @Id long id;
|
||||
|
||||
@@ -132,64 +104,6 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
|
||||
@Column(name = "claimKey", nullable = false)
|
||||
Map<String, String> labelsToKeys;
|
||||
|
||||
/** Indicates that this is a shard rather than a "full" list. */
|
||||
@Ignore @Transient boolean isShard = false;
|
||||
|
||||
private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2);
|
||||
|
||||
private static Optional<ClaimsListShard> loadClaimsListShard() {
|
||||
// Find the most recent revision.
|
||||
Key<ClaimsListRevision> revisionKey = getCurrentRevision();
|
||||
if (revisionKey == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
Map<String, String> combinedLabelsToKeys = new HashMap<>();
|
||||
DateTime creationTime = START_OF_TIME;
|
||||
// Grab all of the keys for the shards that belong to the current revision.
|
||||
final List<Key<ClaimsListShard>> shardKeys =
|
||||
auditedOfy().load().type(ClaimsListShard.class).ancestor(revisionKey).keys().list();
|
||||
|
||||
List<ClaimsListShard> shards;
|
||||
try {
|
||||
// Load all of the shards concurrently, each in a separate transaction.
|
||||
shards =
|
||||
Concurrent.transform(
|
||||
shardKeys,
|
||||
key ->
|
||||
ofyTm()
|
||||
.transactNewReadOnly(
|
||||
() -> {
|
||||
ClaimsListShard claimsListShard = auditedOfy().load().key(key).now();
|
||||
checkState(
|
||||
claimsListShard != null,
|
||||
"Key not found when loading claims list shards.");
|
||||
return claimsListShard;
|
||||
}));
|
||||
} catch (UncheckedExecutionException e) {
|
||||
// We retry on IllegalStateException. However, there's a checkState inside the
|
||||
// Concurrent.transform, so if it's thrown it'll be wrapped in an
|
||||
// UncheckedExecutionException. We want to unwrap it so it's caught by the retrier.
|
||||
if (e.getCause() != null) {
|
||||
throwIfUnchecked(e.getCause());
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Combine the shards together and return the concatenated ClaimsList.
|
||||
if (!shards.isEmpty()) {
|
||||
creationTime = shards.get(0).creationTime;
|
||||
for (ClaimsListShard shard : shards) {
|
||||
combinedLabelsToKeys.putAll(shard.labelsToKeys);
|
||||
checkState(
|
||||
creationTime.equals(shard.creationTime),
|
||||
"Inconsistent claims list shard creation times.");
|
||||
}
|
||||
}
|
||||
|
||||
return Optional.of(create(creationTime, ImmutableMap.copyOf(combinedLabelsToKeys)));
|
||||
}
|
||||
|
||||
/** Returns the revision id of this claims list, or throws exception if it is null. */
|
||||
public Long getRevisionId() {
|
||||
checkState(
|
||||
@@ -227,80 +141,14 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
|
||||
return labelsToKeys.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the Claims list to Datastore by writing the new shards in a series of transactions,
|
||||
* switching over to using them atomically, then deleting the old ones.
|
||||
*/
|
||||
void saveToDatastore() {
|
||||
saveToDatastore(SHARD_SIZE);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
void saveToDatastore(int shardSize) {
|
||||
// Figure out what the next versionId should be based on which ones already exist.
|
||||
final Key<ClaimsListRevision> oldRevision = getCurrentRevision();
|
||||
final Key<ClaimsListRevision> parentKey = ClaimsListRevision.createKey();
|
||||
|
||||
// Save the ClaimsList shards in separate transactions.
|
||||
Concurrent.transform(
|
||||
CollectionUtils.partitionMap(labelsToKeys, shardSize),
|
||||
(final ImmutableMap<String, String> labelsToKeysShard) ->
|
||||
ofyTm()
|
||||
.transact(
|
||||
() -> {
|
||||
ClaimsListShard shard = create(creationTime, labelsToKeysShard);
|
||||
shard.isShard = true;
|
||||
shard.parent = parentKey;
|
||||
auditedOfy().saveWithoutBackup().entity(shard);
|
||||
return shard;
|
||||
}));
|
||||
|
||||
// Persist the new revision, thus causing the newly created shards to go live.
|
||||
ofyTm()
|
||||
.transact(
|
||||
() -> {
|
||||
verify(
|
||||
(getCurrentRevision() == null && oldRevision == null)
|
||||
|| getCurrentRevision().equals(oldRevision),
|
||||
"Registries' ClaimsList was updated by someone else while attempting to update.");
|
||||
auditedOfy().saveWithoutBackup().entity(ClaimsListSingleton.create(parentKey));
|
||||
// Delete the old ClaimsListShard entities.
|
||||
if (oldRevision != null) {
|
||||
auditedOfy()
|
||||
.deleteWithoutBackup()
|
||||
.keys(
|
||||
auditedOfy()
|
||||
.load()
|
||||
.type(ClaimsListShard.class)
|
||||
.ancestor(oldRevision)
|
||||
.keys());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public static ClaimsListShard create(
|
||||
DateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
|
||||
ClaimsListShard instance = new ClaimsListShard();
|
||||
public static ClaimsList create(DateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
|
||||
ClaimsList instance = new ClaimsList();
|
||||
instance.id = allocateId();
|
||||
instance.creationTime = checkNotNull(tmdbGenerationTime);
|
||||
instance.labelsToKeys = checkNotNull(labelsToKeys);
|
||||
return instance;
|
||||
}
|
||||
|
||||
/** Return a single logical instance that combines all Datastore shards. */
|
||||
static Optional<ClaimsListShard> getFromDatastore() {
|
||||
return LOADER_RETRIER.callWithRetry(
|
||||
ClaimsListShard::loadClaimsListShard, IllegalStateException.class);
|
||||
}
|
||||
|
||||
/** As a safety mechanism, fail if someone tries to save this class directly. */
|
||||
@OnSave
|
||||
void disallowUnshardedSaves() {
|
||||
if (!isShard) {
|
||||
throw new UnshardedSaveException();
|
||||
}
|
||||
}
|
||||
|
||||
/** Virtual parent entity for claims list shards of a specific revision. */
|
||||
@Entity
|
||||
@VirtualEntity
|
||||
@@ -354,6 +202,6 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
|
||||
return singleton == null ? null : singleton.activeRevision;
|
||||
}
|
||||
|
||||
/** Exception when trying to directly save a {@link ClaimsListShard} without sharding. */
|
||||
/** Exception when trying to directly save a {@link ClaimsList} without sharding. */
|
||||
public static class UnshardedSaveException extends RuntimeException {}
|
||||
}
|
||||
+12
-10
@@ -15,22 +15,23 @@
|
||||
package google.registry.model.tmch;
|
||||
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import java.util.Optional;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
/** Data access object for {@link ClaimsListShard}. */
|
||||
public class ClaimsListSqlDao {
|
||||
/** Data access object for {@link ClaimsList}. */
|
||||
public class ClaimsListDao {
|
||||
|
||||
/** Saves the given {@link ClaimsListShard} to Cloud SQL. */
|
||||
static void save(ClaimsListShard claimsList) {
|
||||
/** Saves the given {@link ClaimsList} to Cloud SQL. */
|
||||
public static void save(ClaimsList claimsList) {
|
||||
jpaTm().transact(() -> jpaTm().insert(claimsList));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent revision of the {@link ClaimsListShard} in SQL or an empty list if it
|
||||
* Returns the most recent revision of the {@link ClaimsList} in SQL or an empty list if it
|
||||
* doesn't exist.
|
||||
*/
|
||||
static Optional<ClaimsListShard> get() {
|
||||
public static ClaimsList get() {
|
||||
return jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
@@ -42,12 +43,13 @@ public class ClaimsListSqlDao {
|
||||
.query(
|
||||
"FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
|
||||
+ " :revisionId",
|
||||
ClaimsListShard.class)
|
||||
ClaimsList.class)
|
||||
.setParameter("revisionId", revisionId)
|
||||
.getResultStream()
|
||||
.findFirst();
|
||||
});
|
||||
})
|
||||
.orElse(ClaimsList.create(START_OF_TIME, ImmutableMap.of()));
|
||||
}
|
||||
|
||||
private ClaimsListSqlDao() {}
|
||||
private ClaimsListDao() {}
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.tmch;
|
||||
|
||||
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
|
||||
import static google.registry.model.CacheUtils.tryMemoizeWithExpiration;
|
||||
import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.MapDifference;
|
||||
import com.google.common.collect.Maps;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* DAO for {@link ClaimsListShard} objects that handles the branching paths for SQL and Datastore.
|
||||
*
|
||||
* <p>For write actions, this class will perform the action against Cloud SQL then, after that
|
||||
* success or failure, against Datastore. If Datastore fails, an error is logged (but not thrown).
|
||||
*
|
||||
* <p>For read actions, we will log if the two databases have different values (or if the retrieval
|
||||
* from Datastore fails).
|
||||
*/
|
||||
public class ClaimsListDualDatabaseDao {
|
||||
|
||||
/** In-memory cache for claims list. */
|
||||
@NonFinalForTesting
|
||||
private static Supplier<ClaimsListShard> claimsListCache =
|
||||
tryMemoizeWithExpiration(
|
||||
getDomainLabelListCacheDuration(), ClaimsListDualDatabaseDao::getUncached);
|
||||
|
||||
/**
|
||||
* Saves the given {@link ClaimsListShard} to both the primary and secondary databases, logging
|
||||
* and skipping errors in Datastore.
|
||||
*/
|
||||
public static void save(ClaimsListShard claimsList) {
|
||||
ClaimsListSqlDao.save(claimsList);
|
||||
suppressExceptionUnlessInTest(
|
||||
claimsList::saveToDatastore, "Error saving ClaimsListShard to Datastore.");
|
||||
}
|
||||
|
||||
/** Returns the most recent revision of the {@link ClaimsListShard}, from cache. */
|
||||
public static ClaimsListShard get() {
|
||||
return claimsListCache.get();
|
||||
}
|
||||
|
||||
/** Retrieves and compares the latest revision from the databases. */
|
||||
private static ClaimsListShard getUncached() {
|
||||
Optional<ClaimsListShard> cloudSqlResult = ClaimsListSqlDao.get();
|
||||
suppressExceptionUnlessInTest(
|
||||
() -> {
|
||||
Optional<ClaimsListShard> datastoreResult = ClaimsListShard.getFromDatastore();
|
||||
compareClaimsLists(cloudSqlResult, datastoreResult);
|
||||
},
|
||||
"Error loading ClaimsListShard from Datastore.");
|
||||
return cloudSqlResult.orElse(ClaimsListShard.create(START_OF_TIME, ImmutableMap.of()));
|
||||
}
|
||||
|
||||
private static void compareClaimsLists(
|
||||
Optional<ClaimsListShard> maybeCloudSql, Optional<ClaimsListShard> maybeDatastore) {
|
||||
if (maybeCloudSql.isPresent() && !maybeDatastore.isPresent()) {
|
||||
throw new IllegalStateException("Claims list found in Cloud SQL but not in Datastore.");
|
||||
}
|
||||
if (!maybeCloudSql.isPresent() && maybeDatastore.isPresent()) {
|
||||
throw new IllegalStateException("Claims list found in Datastore but not in Cloud SQL.");
|
||||
}
|
||||
if (!maybeCloudSql.isPresent()) {
|
||||
return;
|
||||
}
|
||||
ClaimsListShard sqlList = maybeCloudSql.get();
|
||||
ClaimsListShard datastoreList = maybeDatastore.get();
|
||||
MapDifference<String, String> diff =
|
||||
Maps.difference(sqlList.labelsToKeys, datastoreList.getLabelsToKeys());
|
||||
if (!diff.areEqual()) {
|
||||
if (diff.entriesDiffering().size()
|
||||
+ diff.entriesOnlyOnRight().size()
|
||||
+ diff.entriesOnlyOnLeft().size()
|
||||
> 10) {
|
||||
throw new IllegalStateException(
|
||||
String.format(
|
||||
"Unequal claims lists detected, Datastore list with revision id %d has %d"
|
||||
+ " different records than the current Cloud SQL list.",
|
||||
datastoreList.getRevisionId(), diff.entriesDiffering().size()));
|
||||
} else {
|
||||
StringBuilder diffMessage = new StringBuilder("Unequal claims lists detected:\n");
|
||||
diff.entriesDiffering()
|
||||
.forEach(
|
||||
(label, valueDiff) ->
|
||||
diffMessage.append(
|
||||
String.format(
|
||||
"Domain label %s has key %s in Cloud SQL and key %s "
|
||||
+ "in Datastore.\n",
|
||||
label, valueDiff.leftValue(), valueDiff.rightValue())));
|
||||
diff.entriesOnlyOnLeft()
|
||||
.forEach(
|
||||
(label, valueDiff) ->
|
||||
diffMessage.append(
|
||||
String.format(
|
||||
"Domain label %s with key %s only appears in Cloud SQL.\n",
|
||||
label, valueDiff)));
|
||||
diff.entriesOnlyOnRight()
|
||||
.forEach(
|
||||
(label, valueDiff) ->
|
||||
diffMessage.append(
|
||||
String.format(
|
||||
"Domain label %s with key %s only appears in Datastore.\n",
|
||||
label, valueDiff)));
|
||||
throw new IllegalStateException(diffMessage.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ClaimsListDualDatabaseDao() {}
|
||||
}
|
||||
+1
-1
@@ -37,7 +37,7 @@ import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
|
||||
import google.registry.model.ofy.DatastoreTransactionManager;
|
||||
import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
|
||||
import google.registry.persistence.JpaRetries;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.util.Clock;
|
||||
|
||||
@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
|
||||
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import java.util.List;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
@@ -34,11 +34,11 @@ import org.joda.time.DateTime;
|
||||
public class ClaimsListParser {
|
||||
|
||||
/**
|
||||
* Converts the lines from the DNL CSV file into a {@link ClaimsListShard} object.
|
||||
* Converts the lines from the DNL CSV file into a {@link ClaimsList} object.
|
||||
*
|
||||
* <p>Please note that this does <b>not</b> insert the object into Datastore.
|
||||
*/
|
||||
public static ClaimsListShard parse(List<String> lines) {
|
||||
public static ClaimsList parse(List<String> lines) {
|
||||
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
|
||||
|
||||
// First line: <version>,<DNL List creation datetime>
|
||||
@@ -74,6 +74,6 @@ public class ClaimsListParser {
|
||||
builder.put(label, lookupKey);
|
||||
}
|
||||
|
||||
return ClaimsListShard.create(creationTime, builder.build());
|
||||
return ClaimsList.create(creationTime, builder.build());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,8 +18,8 @@ import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.keyring.api.KeyModule.Key;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.auth.Auth;
|
||||
import java.io.IOException;
|
||||
@@ -55,8 +55,8 @@ public final class TmchDnlAction implements Runnable {
|
||||
} catch (SignatureException | IOException | PGPException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
ClaimsListShard claims = ClaimsListParser.parse(lines);
|
||||
ClaimsListDualDatabaseDao.save(claims);
|
||||
ClaimsList claims = ClaimsListParser.parse(lines);
|
||||
ClaimsListDao.save(claims);
|
||||
logger.atInfo().log(
|
||||
"Inserted %,d claims into the DB(s), created at %s",
|
||||
claims.size(), claims.getTmdbGenerationTime());
|
||||
|
||||
@@ -21,8 +21,8 @@ import com.beust.jcommander.Parameter;
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.tools.params.PathParameter;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
@@ -44,7 +44,7 @@ final class GetClaimsListCommand implements CommandWithRemoteApi {
|
||||
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
ClaimsListShard cl = checkNotNull(ClaimsListDualDatabaseDao.get(), "Couldn't load ClaimsList");
|
||||
ClaimsList cl = checkNotNull(ClaimsListDao.get(), "Couldn't load ClaimsList");
|
||||
String csv = Joiner.on('\n').withKeyValueSeparator(",").join(cl.getLabelsToKeys()) + "\n";
|
||||
Files.asCharSink(output.toFile(), UTF_8).write(csv);
|
||||
}
|
||||
|
||||
@@ -21,15 +21,15 @@ import com.beust.jcommander.Parameter;
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.tmch.ClaimsListParser;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** A command to upload a {@link ClaimsListShard}. */
|
||||
/** A command to upload a {@link ClaimsList}. */
|
||||
@Parameters(separators = " =", commandDescription = "Manually upload a new claims list file")
|
||||
final class UploadClaimsListCommand extends ConfirmingCommand implements CommandWithRemoteApi {
|
||||
|
||||
@@ -38,7 +38,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
|
||||
|
||||
private String claimsListFilename;
|
||||
|
||||
private ClaimsListShard claimsList;
|
||||
private ClaimsList claimsList;
|
||||
|
||||
@Override
|
||||
protected void init() throws IOException {
|
||||
@@ -57,7 +57,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
|
||||
|
||||
@Override
|
||||
public String execute() {
|
||||
ClaimsListDualDatabaseDao.save(claimsList);
|
||||
ClaimsListDao.save(claimsList);
|
||||
return String.format("Successfully uploaded claims list %s", claimsListFilename);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
<class>google.registry.model.server.Lock</class>
|
||||
<class>google.registry.model.server.ServerSecret</class>
|
||||
<class>google.registry.model.smd.SignedMarkRevocationList</class>
|
||||
<class>google.registry.model.tmch.ClaimsListShard</class>
|
||||
<class>google.registry.model.tmch.ClaimsList</class>
|
||||
<class>google.registry.model.tmch.TmchCrl</class>
|
||||
<class>google.registry.persistence.transaction.TransactionEntity</class>
|
||||
<class>google.registry.schema.domain.RegistryLock</class>
|
||||
|
||||
Reference in New Issue
Block a user