mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-09-12 11:06:29 +00:00
Code Issues Releases Wiki Activity

Stop writing PremiumList to Datastore (#1160)

Browse Source 
* Stop writing PremiumList to Datastore

* Fix formatting

* Format fix

* Rename the DAO

* Fix merge conflicts and add comment
This commit is contained in:
sarahcaseybot
2021-05-14 16:13:05 -04:00
committed by GitHub
parent e09138645f
commit 93dc812ea2
34 changed files with 142 additions and 1295 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/google/registry/export/ExportPremiumTermsAction.java
+4 -3
View File
@@ -32,12 +32,12 @@ import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config;
import google.registry.model.registry.Registry;
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.RequestParameters;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.schema.tld.PremiumListDao;
import google.registry.storage.drive.DriveConnection;
import java.io.IOException;
import java.util.Optional;
@@ -139,9 +139,10 @@ public class ExportPremiumTermsAction implements Runnable {
private String getFormattedPremiumTerms(Registry registry) {
String premiumListName = registry.getPremiumList().getName();
checkState(
PremiumListDualDao.exists(premiumListName), "Could not load premium list for " + tld);
PremiumListDao.getLatestRevision(premiumListName).isPresent(),
"Could not load premium list for " + tld);
SortedSet<String> premiumTerms =
Streams.stream(PremiumListDualDao.loadAllPremiumListEntries(premiumListName))
Streams.stream(PremiumListDao.loadAllPremiumListEntries(premiumListName))
.map(PremiumListEntry::toString)
.collect(ImmutableSortedSet.toImmutableSortedSet(String::compareTo));
core/src/main/java/google/registry/model/OteAccountBuilder.java
+2 -2
View File
@@ -39,8 +39,8 @@ import google.registry.model.registrar.RegistrarContact;
import google.registry.model.registry.Registry;
import google.registry.model.registry.Registry.TldState;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.persistence.VKey;
import google.registry.schema.tld.PremiumListDao;
import google.registry.util.CidrAddressBlock;
import java.util.Collection;
import java.util.Optional;
@@ -295,7 +295,7 @@ public final class OteAccountBuilder {
boolean isEarlyAccess,
int roidSuffix) {
String tldNameAlphaNumerical = tldName.replaceAll("[^a-z0-9]", "");
Optional<PremiumList> premiumList = PremiumListDualDao.getLatestRevision(DEFAULT_PREMIUM_LIST);
Optional<PremiumList> premiumList = PremiumListDao.getLatestRevision(DEFAULT_PREMIUM_LIST);
checkState(premiumList.isPresent(), "Couldn't find premium list %s.", DEFAULT_PREMIUM_LIST);
Registry.Builder builder =
new Registry.Builder()
core/src/main/java/google/registry/model/pricing/StaticPremiumListPricingEngine.java
+3 -2
View File
@@ -19,7 +19,7 @@ import static google.registry.util.DomainNameUtils.getTldFromDomainName;
import com.google.common.net.InternetDomainName;
import google.registry.model.registry.Registry;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.schema.tld.PremiumListDao;
import java.util.Optional;
import javax.inject.Inject;
import org.joda.money.Money;
@@ -38,7 +38,8 @@ public final class StaticPremiumListPricingEngine implements PremiumPricingEngin
String tld = getTldFromDomainName(fullyQualifiedDomainName);
String label = InternetDomainName.from(fullyQualifiedDomainName).parts().get(0);
Registry registry = Registry.get(checkNotNull(tld, "tld"));
Optional<Money> premiumPrice = PremiumListDualDao.getPremiumPrice(label, registry);
Optional<Money> premiumPrice =
PremiumListDao.getPremiumPrice(registry.getPremiumList().getName(), label);
return DomainPrices.create(
premiumPrice.isPresent(),
premiumPrice.orElse(registry.getStandardCreateCost()),
core/src/main/java/google/registry/model/registry/label/PremiumList.java
+2 -2
View File
@@ -36,7 +36,7 @@ import google.registry.model.annotations.ReportedOn;
import google.registry.model.registry.Registry;
import google.registry.schema.replay.DatastoreOnlyEntity;
import google.registry.schema.replay.NonReplicatedEntity;
import google.registry.schema.tld.PremiumListSqlDao;
import google.registry.schema.tld.PremiumListDao;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
@@ -173,7 +173,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
* <p>Note that this is lazily loaded and thus will throw a {@link LazyInitializationException} if
* used outside the transaction in which the given entity was loaded. You generally should not be
* using this anyway as it's inefficient to load all of the PremiumEntry rows if you don't need
* them. To check prices, use {@link PremiumListSqlDao#getPremiumPrice} instead.
* them. To check prices, use {@link PremiumListDao#getPremiumPrice} instead.
*/
@Nullable
public ImmutableMap<String, BigDecimal> getLabelsToPrices() {
core/src/main/java/google/registry/model/registry/label/PremiumListDatastoreDao.java
-375
View File
@@ -1,375 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.registry.label;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.collect.Iterables.partition;
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
import static google.registry.config.RegistryConfig.getSingletonCachePersistDuration;
import static google.registry.config.RegistryConfig.getStaticPremiumListMaxCachedEntries;
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.model.registry.label.DomainLabelMetrics.PremiumListCheckOutcome.BLOOM_FILTER_NEGATIVE;
import static google.registry.model.registry.label.DomainLabelMetrics.PremiumListCheckOutcome.CACHED_NEGATIVE;
import static google.registry.model.registry.label.DomainLabelMetrics.PremiumListCheckOutcome.CACHED_POSITIVE;
import static google.registry.model.registry.label.DomainLabelMetrics.PremiumListCheckOutcome.UNCACHED_NEGATIVE;
import static google.registry.model.registry.label.DomainLabelMetrics.PremiumListCheckOutcome.UNCACHED_POSITIVE;
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static org.joda.time.DateTimeZone.UTC;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Streams;
import com.googlecode.objectify.Key;
import google.registry.model.registry.label.DomainLabelMetrics.PremiumListCheckOutcome;
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
import google.registry.model.registry.label.PremiumList.PremiumListRevision;
import google.registry.persistence.VKey;
import google.registry.util.NonFinalForTesting;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import org.joda.money.Money;
import org.joda.time.DateTime;
import org.joda.time.Duration;
/**
* DAO for {@link PremiumList} objects stored in Datastore.
*
* <p>This class handles both the mapping from string to Datastore-level PremiumList objects as well
* as the mapping from PremiumList objects to the contents of those premium lists in the Datastore
* world. Specifically, this deals with retrieving the most recent revision for a given list and
* retrieving (or writing/deleting) all entries associated with that particular revision. The {@link
* PremiumList} object itself, in the Datastore world, does not store the premium pricing data.
*/
public class PremiumListDatastoreDao {
/** The number of premium list entry entities that are created and deleted per batch. */
private static final int TRANSACTION_BATCH_SIZE = 200;
/**
* In-memory cache for premium lists.
*
* <p>This is cached for a shorter duration because we need to periodically reload this entity to
* check if a new revision has been published, and if so, then use that.
*
* <p>We also cache the absence of premium lists with a given name to avoid unnecessary pointless
* lookups. Note that this cache is only applicable to PremiumList objects stored in Datastore.
*/
@NonFinalForTesting
static LoadingCache<String, Optional<PremiumList>> premiumListCache =
createPremiumListCache(getDomainLabelListCacheDuration());
@VisibleForTesting
public static void setPremiumListCacheForTest(Optional<Duration> expiry) {
Duration effectiveExpiry = expiry.orElse(getSingletonCachePersistDuration());
premiumListCache = createPremiumListCache(effectiveExpiry);
}
@VisibleForTesting
public static LoadingCache<String, Optional<PremiumList>> createPremiumListCache(
Duration cachePersistDuration) {
return CacheBuilder.newBuilder()
.expireAfterWrite(java.time.Duration.ofMillis(cachePersistDuration.getMillis()))
.build(
new CacheLoader<String, Optional<PremiumList>>() {
@Override
public Optional<PremiumList> load(final String name) {
return tm().doTransactionless(() -> getLatestRevisionUncached(name));
}
});
}
/**
* In-memory cache for {@link PremiumListRevision}s, used for retrieving Bloom filters quickly.
*
* <p>This is cached for a long duration (essentially indefinitely) because a given {@link
* PremiumListRevision} is immutable and cannot ever be changed once created, so its cache need
* not ever expire.
*/
static final LoadingCache<Key<PremiumListRevision>, PremiumListRevision>
premiumListRevisionsCache =
CacheBuilder.newBuilder()
.expireAfterWrite(
java.time.Duration.ofMillis(getSingletonCachePersistDuration().getMillis()))
.build(
new CacheLoader<Key<PremiumListRevision>, PremiumListRevision>() {
@Override
public PremiumListRevision load(final Key<PremiumListRevision> revisionKey) {
return ofyTm()
.doTransactionless(() -> auditedOfy().load().key(revisionKey).now());
}
});
/**
* In-memory cache for {@link PremiumListEntry}s for a given label and {@link PremiumListRevision}
*
* <p>Because the PremiumList itself makes up part of the PremiumListRevision's key, this is
* specific to a given premium list. Premium list entries might not be present, as indicated by
* the Optional wrapper, and we want to cache that as well.
*
* <p>This is cached for a long duration (essentially indefinitely) because a given {@link
* PremiumListRevision} and its child {@link PremiumListEntry}s are immutable and cannot ever be
* changed once created, so the cache need not ever expire.
*
* <p>A maximum size is set here on the cache because it can potentially grow too big to fit in
* memory if there are a large number of distinct premium list entries being queried (both those
* that exist, as well as those that might exist according to the Bloom filter, must be cached).
* The entries judged least likely to be accessed again will be evicted first.
*/
@NonFinalForTesting
static LoadingCache<Key<PremiumListEntry>, Optional<PremiumListEntry>> premiumListEntriesCache =
createPremiumListEntriesCache(getSingletonCachePersistDuration());
@VisibleForTesting
public static void setPremiumListEntriesCacheForTest(Optional<Duration> expiry) {
Duration effectiveExpiry = expiry.orElse(getSingletonCachePersistDuration());
premiumListEntriesCache = createPremiumListEntriesCache(effectiveExpiry);
}
@VisibleForTesting
static LoadingCache<Key<PremiumListEntry>, Optional<PremiumListEntry>>
createPremiumListEntriesCache(Duration cachePersistDuration) {
return CacheBuilder.newBuilder()
.expireAfterWrite(java.time.Duration.ofMillis(cachePersistDuration.getMillis()))
.maximumSize(getStaticPremiumListMaxCachedEntries())
.build(
new CacheLoader<Key<PremiumListEntry>, Optional<PremiumListEntry>>() {
@Override
public Optional<PremiumListEntry> load(final Key<PremiumListEntry> entryKey) {
return ofyTm()
.doTransactionless(
() -> Optional.ofNullable(auditedOfy().load().key(entryKey).now()));
}
});
}
public static Optional<PremiumList> getLatestRevision(String name) {
return premiumListCache.getUnchecked(name);
}
/**
* Returns the premium price for the specified list, label, and TLD, or absent if the label is not
* premium.
*/
public static Optional<Money> getPremiumPrice(String premiumListName, String label, String tld) {
DateTime startTime = DateTime.now(UTC);
Optional<PremiumList> maybePremumList = getLatestRevision(premiumListName);
if (!maybePremumList.isPresent()) {
return Optional.empty();
}
PremiumList premiumList = maybePremumList.get();
// If we're dealing with a list from SQL, reload from Datastore if necessary
if (premiumList.getRevisionKey() == null) {
Optional<PremiumList> fromDatastore = getLatestRevision(premiumList.getName());
if (fromDatastore.isPresent()) {
premiumList = fromDatastore.get();
} else {
return Optional.empty();
}
}
PremiumListRevision revision;
try {
revision = premiumListRevisionsCache.get(premiumList.getRevisionKey());
} catch (InvalidCacheLoadException | ExecutionException e) {
throw new RuntimeException(
"Could not load premium list revision " + premiumList.getRevisionKey(), e);
}
checkState(
revision.getProbablePremiumLabels() != null,
"Probable premium labels Bloom filter is null on revision '%s'",
premiumList.getRevisionKey());
CheckResults checkResults = checkStatus(revision, label);
DomainLabelMetrics.recordPremiumListCheckOutcome(
tld,
premiumList.getName(),
checkResults.checkOutcome(),
DateTime.now(UTC).getMillis() - startTime.getMillis());
return checkResults.premiumPrice();
}
/**
* Persists a new or updated PremiumList object and its descendant entities to Datastore.
*
* <p>The flow here is: save the new premium list entries parented on that revision entity,
* save/update the PremiumList, and then delete the old premium list entries associated with the
* old revision.
*
* <p>This is the only valid way to save these kinds of entities!
*/
public static PremiumList save(String name, List<String> inputData) {
PremiumList premiumList = new PremiumList.Builder().setName(name).build();
ImmutableMap<String, PremiumListEntry> premiumListEntries = premiumList.parse(inputData);
final Optional<PremiumList> oldPremiumList = getLatestRevisionUncached(premiumList.getName());
// Create the new revision (with its Bloom filter) and parent the entries on it.
final PremiumListRevision newRevision =
PremiumListRevision.create(premiumList, premiumListEntries.keySet());
final Key<PremiumListRevision> newRevisionKey = Key.create(newRevision);
ImmutableSet<PremiumListEntry> parentedEntries =
parentPremiumListEntriesOnRevision(premiumListEntries.values(), newRevisionKey);
// Save the new child entities in a series of transactions.
for (final List<PremiumListEntry> batch : partition(parentedEntries, TRANSACTION_BATCH_SIZE)) {
ofyTm().transactNew(() -> auditedOfy().save().entities(batch));
}
// Save the new PremiumList and revision itself.
return ofyTm()
.transactNew(
() -> {
DateTime now = ofyTm().getTransactionTime();
// Assert that the premium list hasn't been changed since we started this process.
Key<PremiumList> key =
Key.create(getCrossTldKey(), PremiumList.class, premiumList.getName());
Optional<PremiumList> existing =
ofyTm().loadByKeyIfPresent(VKey.createOfy(PremiumList.class, key));
checkOfyFieldsEqual(existing, oldPremiumList);
PremiumList newList =
premiumList
.asBuilder()
.setLastUpdateTime(now)
.setCreationTime(
oldPremiumList.isPresent() ? oldPremiumList.get().creationTime : now)
.setRevision(newRevisionKey)
.build();
auditedOfy().save().entities(newList, newRevision);
premiumListCache.invalidate(premiumList.getName());
return newList;
});
}
public static void delete(PremiumList premiumList) {
ofyTm().transactNew(() -> auditedOfy().delete().entity(premiumList));
if (premiumList.getRevisionKey() == null) {
return;
}
for (final List<Key<PremiumListEntry>> batch :
partition(
auditedOfy()
.load()
.type(PremiumListEntry.class)
.ancestor(premiumList.revisionKey)
.keys(),
TRANSACTION_BATCH_SIZE)) {
ofyTm().transactNew(() -> auditedOfy().delete().keys(batch));
batch.forEach(premiumListEntriesCache::invalidate);
}
ofyTm().transactNew(() -> auditedOfy().delete().key(premiumList.getRevisionKey()));
premiumListCache.invalidate(premiumList.getName());
premiumListRevisionsCache.invalidate(premiumList.getRevisionKey());
}
/** Re-parents the given {@link PremiumListEntry}s on the given {@link PremiumListRevision}. */
@VisibleForTesting
public static ImmutableSet<PremiumListEntry> parentPremiumListEntriesOnRevision(
Iterable<PremiumListEntry> entries, final Key<PremiumListRevision> revisionKey) {
return Streams.stream(entries)
.map((PremiumListEntry entry) -> entry.asBuilder().setParent(revisionKey).build())
.collect(toImmutableSet());
}
/**
* Returns all {@link PremiumListEntry PremiumListEntries} in the given {@code premiumList}.
*
* <p>This is an expensive operation and should only be used when the entire list is required.
*/
public static Iterable<PremiumListEntry> loadPremiumListEntriesUncached(PremiumList premiumList) {
return auditedOfy()
.load()
.type(PremiumListEntry.class)
.ancestor(premiumList.revisionKey)
.iterable();
}
private static Optional<PremiumList> getLatestRevisionUncached(String name) {
return Optional.ofNullable(
auditedOfy().load().key(Key.create(getCrossTldKey(), PremiumList.class, name)).now());
}
private static void checkOfyFieldsEqual(
Optional<PremiumList> oneOptional, Optional<PremiumList> twoOptional) {
if (!oneOptional.isPresent()) {
checkState(!twoOptional.isPresent(), "Premium list concurrently deleted");
return;
} else {
checkState(twoOptional.isPresent(), "Premium list concurrently deleted");
}
PremiumList one = oneOptional.get();
PremiumList two = twoOptional.get();
checkState(
Objects.equals(one.revisionKey, two.revisionKey),
"Premium list revision key concurrently edited");
checkState(Objects.equals(one.name, two.name), "Premium list name concurrently edited");
checkState(Objects.equals(one.parent, two.parent), "Premium list parent concurrently edited");
checkState(
Objects.equals(one.creationTime, two.creationTime),
"Premium list creation time concurrently edited");
}
private static CheckResults checkStatus(PremiumListRevision premiumListRevision, String label) {
if (!premiumListRevision.getProbablePremiumLabels().mightContain(label)) {
return CheckResults.create(BLOOM_FILTER_NEGATIVE, Optional.empty());
}
Key<PremiumListEntry> entryKey =
Key.create(Key.create(premiumListRevision), PremiumListEntry.class, label);
try {
// getIfPresent() returns null if the key is not in the cache
Optional<PremiumListEntry> entry = premiumListEntriesCache.getIfPresent(entryKey);
if (entry != null) {
if (entry.isPresent()) {
return CheckResults.create(CACHED_POSITIVE, Optional.of(entry.get().getValue()));
} else {
return CheckResults.create(CACHED_NEGATIVE, Optional.empty());
}
}
entry = premiumListEntriesCache.get(entryKey);
if (entry.isPresent()) {
return CheckResults.create(UNCACHED_POSITIVE, Optional.of(entry.get().getValue()));
} else {
return CheckResults.create(UNCACHED_NEGATIVE, Optional.empty());
}
} catch (InvalidCacheLoadException | ExecutionException e) {
throw new RuntimeException("Could not load premium list entry " + entryKey, e);
}
}
/** Value type class used by {@link #checkStatus} to return the results of a premiumness check. */
@AutoValue
abstract static class CheckResults {
static CheckResults create(PremiumListCheckOutcome checkOutcome, Optional<Money> premiumPrice) {
return new AutoValue_PremiumListDatastoreDao_CheckResults(checkOutcome, premiumPrice);
}
abstract PremiumListCheckOutcome checkOutcome();
abstract Optional<Money> premiumPrice();
}
private PremiumListDatastoreDao() {}
}
core/src/main/java/google/registry/model/registry/label/PremiumListDualDao.java
-140
View File
@@ -1,140 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.registry.label;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
import com.google.common.collect.Streams;
import google.registry.model.registry.Registry;
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
import google.registry.schema.tld.PremiumListSqlDao;
import java.util.List;
import java.util.Optional;
import org.joda.money.BigMoney;
import org.joda.money.CurrencyUnit;
import org.joda.money.Money;
/**
* DAO for {@link PremiumList} objects that handles the branching paths for SQL and Datastore.
*
* <p>For write actions, this class will perform the action against Cloud SQL then, after that
* success or failure, against Datastore. If Datastore fails, an error is logged (but not thrown).
*
* <p>For read actions, when retrieving a price, we will log if the primary and secondary databases
* have different values (or if the retrieval from Datastore fails).
*/
public class PremiumListDualDao {
/**
* Retrieves from Cloud SQL and returns the most recent premium list with the given name, or
* absent if no such list exists.
*/
public static Optional<PremiumList> getLatestRevision(String premiumListName) {
return PremiumListSqlDao.getLatestRevision(premiumListName);
}
/**
* Returns the premium price for the specified label and registry.
*
* <p>Returns absent if the label is not premium or there is no premium list for this registry.
*
* <p>Retrieves the price from both primary and secondary databases, and logs in the event of a
* failure in Datastore (but does not throw an exception).
*/
public static Optional<Money> getPremiumPrice(String label, Registry registry) {
if (registry.getPremiumList() == null) {
return Optional.empty();
}
String premiumListName = registry.getPremiumList().getName();
Optional<Money> primaryResult;
primaryResult = PremiumListSqlDao.getPremiumPrice(premiumListName, label);
// Also load the value from Datastore, compare the two results, and log if different.
suppressExceptionUnlessInTest(
() -> {
Optional<Money> secondaryResult =
PremiumListDatastoreDao.getPremiumPrice(premiumListName, label, registry.getTldStr());
checkState(
primaryResult.equals(secondaryResult),
"Unequal prices for domain %s.%s from primary SQL DB (%s) and secondary Datastore db"
+ " (%s).",
label,
registry.getTldStr(),
primaryResult,
secondaryResult);
},
String.format(
"Error loading price of domain %s.%s from Datastore.", label, registry.getTldStr()));
return primaryResult;
}
/**
* Saves the given list data to both primary and secondary databases.
*
* <p>Logs but doesn't throw an exception in the event of a failure when writing to Datastore.
*/
public static PremiumList save(String name, List<String> inputData) {
PremiumList result = PremiumListSqlDao.save(name, inputData);
suppressExceptionUnlessInTest(
() -> PremiumListDatastoreDao.save(name, inputData),
"Error when saving premium list to Datastore.");
return result;
}
/**
* Deletes the premium list.
*
* <p>Logs but doesn't throw an exception in the event of a failure when deleting from Datastore.
*/
public static void delete(PremiumList premiumList) {
PremiumListSqlDao.delete(premiumList);
suppressExceptionUnlessInTest(
() -> PremiumListDatastoreDao.delete(premiumList),
"Error when deleting premium list from Datastore.");
}
/** Returns whether or not there exists a premium list with the given name. */
public static boolean exists(String premiumListName) {
// It may seem like overkill, but loading the list has ways been the way we check existence and
// given that we usually load the list around the time we check existence, we'll hit the cache
return PremiumListSqlDao.getLatestRevision(premiumListName).isPresent();
}
/**
* Returns all {@link PremiumListEntry PremiumListEntries} in the list with the given name.
*
* <p>This is an expensive operation and should only be used when the entire list is required.
*/
public static Iterable<PremiumListEntry> loadAllPremiumListEntries(String premiumListName) {
PremiumList premiumList =
getLatestRevision(premiumListName)
.orElseThrow(
() ->
new IllegalArgumentException(
String.format("No premium list with name %s.", premiumListName)));
CurrencyUnit currencyUnit = premiumList.getCurrency();
return Streams.stream(PremiumListSqlDao.loadPremiumListEntriesUncached(premiumList))
.map(
premiumEntry ->
new PremiumListEntry.Builder()
.setPrice(BigMoney.of(currencyUnit, premiumEntry.getPrice()).toMoney())
.setLabel(premiumEntry.getDomainLabel())
.build())
.collect(toImmutableList());
}
private PremiumListDualDao() {}
}
core/src/main/java/google/registry/schema/tld/PremiumListSqlDao.java → core/src/main/java/google/registry/schema/tld/PremiumListDao.java
+31 -4
View File
@@ -14,6 +14,7 @@
package google.registry.schema.tld;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
import static google.registry.config.RegistryConfig.getSingletonCachePersistDuration;
import static google.registry.config.RegistryConfig.getStaticPremiumListMaxCachedEntries;
@@ -25,6 +26,7 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Streams;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
import google.registry.util.NonFinalForTesting;
@@ -32,6 +34,8 @@ import java.math.BigDecimal;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import org.joda.money.BigMoney;
import org.joda.money.CurrencyUnit;
import org.joda.money.Money;
import org.joda.time.Duration;
@@ -43,7 +47,7 @@ import org.joda.time.Duration;
* {@link PremiumList} object in SQL, and caching these entries so that future lookups can be
* quicker.
*/
public class PremiumListSqlDao {
public class PremiumListDao {
/**
* In-memory cache for premium lists.
@@ -188,7 +192,7 @@ public class PremiumListSqlDao {
*
* <p>This is an expensive operation and should only be used when the entire list is required.
*/
public static Iterable<PremiumEntry> loadPremiumListEntriesUncached(PremiumList premiumList) {
public static Iterable<PremiumEntry> loadPremiumListEntries(PremiumList premiumList) {
return jpaTm()
.transact(
() ->
@@ -220,6 +224,29 @@ public class PremiumListSqlDao {
.findFirst());
}
/**
* Returns all {@link PremiumListEntry PremiumListEntries} in the list with the given name.
*
* <p>This is an expensive operation and should only be used when the entire list is required.
*/
public static Iterable<PremiumListEntry> loadAllPremiumListEntries(String premiumListName) {
PremiumList premiumList =
getLatestRevision(premiumListName)
.orElseThrow(
() ->
new IllegalArgumentException(
String.format("No premium list with name %s.", premiumListName)));
CurrencyUnit currencyUnit = premiumList.getCurrency();
return Streams.stream(loadPremiumListEntries(premiumList))
.map(
premiumEntry ->
new PremiumListEntry.Builder()
.setPrice(BigMoney.of(currencyUnit, premiumEntry.getPrice()).toMoney())
.setLabel(premiumEntry.getDomainLabel())
.build())
.collect(toImmutableList());
}
@AutoValue
abstract static class RevisionIdAndLabel {
abstract long revisionId();
@@ -227,9 +254,9 @@ public class PremiumListSqlDao {
abstract String label();
static RevisionIdAndLabel create(long revisionId, String label) {
return new AutoValue_PremiumListSqlDao_RevisionIdAndLabel(revisionId, label);
return new AutoValue_PremiumListDao_RevisionIdAndLabel(revisionId, label);
}
}
private PremiumListSqlDao() {}
private PremiumListDao() {}
}
core/src/main/java/google/registry/tools/ComparePremiumListsCommand.java
-105
View File
@@ -1,105 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
import com.beust.jcommander.Parameters;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.collect.Streams;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
import google.registry.model.registry.label.PremiumListDatastoreDao;
import google.registry.schema.tld.PremiumEntry;
import google.registry.schema.tld.PremiumListSqlDao;
import java.util.Optional;
import org.joda.money.BigMoney;
/** Command to compare all PremiumLists in Datastore to all PremiumLists in Cloud SQL. */
@Parameters(
separators = " =",
commandDescription = "Compare all the PremiumLists in Datastore to those in Cloud SQL.")
final class ComparePremiumListsCommand implements CommandWithRemoteApi {
@Override
public void run() {
ImmutableSet<String> datastoreLists =
ofyTm().loadAllOf(PremiumList.class).stream()
.map(PremiumList::getName)
.collect(toImmutableSet());
ImmutableSet<String> sqlLists =
jpaTm()
.transact(
() ->
jpaTm().loadAllOf(PremiumList.class).stream()
.map(PremiumList::getName)
.collect(toImmutableSet()));
int listsWithDiffs = 0;
for (String listName : Sets.difference(datastoreLists, sqlLists)) {
listsWithDiffs++;
System.out.printf(
"PremiumList '%s' is present in Datastore, but not in Cloud SQL.%n", listName);
}
for (String listName : Sets.difference(sqlLists, datastoreLists)) {
listsWithDiffs++;
System.out.printf(
"PremiumList '%s' is present in Cloud SQL, but not in Datastore.%n", listName);
}
for (String listName : Sets.intersection(datastoreLists, sqlLists)) {
Optional<PremiumList> sqlList = PremiumListSqlDao.getLatestRevision(listName);
// Datastore and Cloud SQL use different objects to represent premium list entries
// so the best way to compare them is to compare their string representations.
ImmutableSet<String> datastoreListStrings =
Streams.stream(
PremiumListDatastoreDao.loadPremiumListEntriesUncached(
PremiumListDatastoreDao.getLatestRevision(listName).get()))
.map(PremiumListEntry::toString)
.collect(toImmutableSet());
Iterable<PremiumEntry> sqlListEntries =
jpaTm().transact(() -> PremiumListSqlDao.loadPremiumListEntriesUncached(sqlList.get()));
ImmutableSet<String> sqlListStrings =
Streams.stream(sqlListEntries)
.map(
premiumEntry ->
new PremiumListEntry.Builder()
.setPrice(
BigMoney.of(sqlList.get().getCurrency(), premiumEntry.getPrice())
.toMoney())
.setLabel(premiumEntry.getDomainLabel())
.build()
.toString())
.collect(toImmutableSet());
// This will only print out the name of the unequal list. GetPremiumListCommand
// should be used to determine what the actual differences are.
if (!datastoreListStrings.equals(sqlListStrings)) {
listsWithDiffs++;
System.out.printf("PremiumList '%s' has different entries in each database.%n", listName);
}
}
System.out.printf("Found %d unequal list(s).%n", listsWithDiffs);
}
}
core/src/main/java/google/registry/tools/CreateOrUpdatePremiumListCommand.java
+2 -2
View File
@@ -16,7 +16,7 @@ package google.registry.tools;
import com.beust.jcommander.Parameter;
import com.google.common.flogger.FluentLogger;
import google.registry.schema.tld.PremiumListSqlDao;
import google.registry.schema.tld.PremiumListDao;
import google.registry.tools.params.PathParameter;
import java.nio.file.Path;
import java.util.List;
@@ -51,7 +51,7 @@ abstract class CreateOrUpdatePremiumListCommand extends MutatingCommand {
String message = String.format("Saved premium list %s with %d entries", name, inputData.size());
try {
logger.atInfo().log("Saving premium list for TLD %s", name);
PremiumListSqlDao.save(name, inputData);
PremiumListDao.save(name, inputData);
logger.atInfo().log(message);
} catch (Throwable e) {
message = "Unexpected error saving premium list from nomulus tool command";
core/src/main/java/google/registry/tools/CreateOrUpdateTldCommand.java
+2 -2
View File
@@ -31,7 +31,7 @@ import google.registry.model.registry.Registry;
import google.registry.model.registry.Registry.TldState;
import google.registry.model.registry.Registry.TldType;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.schema.tld.PremiumListDao;
import google.registry.tools.params.OptionalStringParameter;
import google.registry.tools.params.TransitionListParameter.BillingCostTransitions;
import google.registry.tools.params.TransitionListParameter.TldStateTransitions;
@@ -344,7 +344,7 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
if (premiumListName != null) {
if (premiumListName.isPresent()) {
Optional<PremiumList> premiumList =
PremiumListDualDao.getLatestRevision(premiumListName.get());
PremiumListDao.getLatestRevision(premiumListName.get());
checkArgument(
premiumList.isPresent(),
String.format("The premium list '%s' doesn't exist", premiumListName.get()));
core/src/main/java/google/registry/tools/CreatePremiumListCommand.java
+2 -2
View File
@@ -25,7 +25,7 @@ import com.google.common.base.Strings;
import com.googlecode.objectify.Key;
import google.registry.model.registry.label.PremiumList;
import google.registry.persistence.VKey;
import google.registry.schema.tld.PremiumListSqlDao;
import google.registry.schema.tld.PremiumListDao;
import google.registry.schema.tld.PremiumListUtils;
import java.nio.file.Files;
@@ -43,7 +43,7 @@ public class CreatePremiumListCommand extends CreateOrUpdatePremiumListCommand {
protected void init() throws Exception {
name = Strings.isNullOrEmpty(name) ? convertFilePathToName(inputFile) : name;
checkArgument(
!PremiumListSqlDao.getLatestRevision(name).isPresent(),
!PremiumListDao.getLatestRevision(name).isPresent(),
"A premium list already exists by this name");
if (!override) {
// refer to CreatePremiumListAction.java
core/src/main/java/google/registry/tools/DeletePremiumListCommand.java
+4 -4
View File
@@ -21,7 +21,7 @@ import com.beust.jcommander.Parameters;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.schema.tld.PremiumListDao;
import javax.annotation.Nullable;
/**
@@ -42,10 +42,10 @@ final class DeletePremiumListCommand extends ConfirmingCommand implements Comman
@Override
protected void init() {
checkArgument(
PremiumListDualDao.exists(name),
PremiumListDao.getLatestRevision(name).isPresent(),
"Cannot delete the premium list %s because it doesn't exist.",
name);
premiumList = PremiumListDualDao.getLatestRevision(name).get();
premiumList = PremiumListDao.getLatestRevision(name).get();
ImmutableSet<String> tldsUsedOn = premiumList.getReferencingTlds();
checkArgument(
tldsUsedOn.isEmpty(),
@@ -60,7 +60,7 @@ final class DeletePremiumListCommand extends ConfirmingCommand implements Comman
@Override
protected String execute() {
PremiumListDualDao.delete(premiumList);
PremiumListDao.delete(premiumList);
return String.format("Deleted premium list '%s'.\n", premiumList.getName());
}
}
core/src/main/java/google/registry/tools/GetPremiumListCommand.java
+3 -3
View File
@@ -18,7 +18,7 @@ import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.collect.Streams;
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.schema.tld.PremiumListDao;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@@ -33,11 +33,11 @@ public class GetPremiumListCommand implements CommandWithRemoteApi {
@Override
public void run() {
for (String premiumListName : mainParameters) {
if (PremiumListDualDao.exists(premiumListName)) {
if (PremiumListDao.getLatestRevision(premiumListName).isPresent()) {
System.out.printf(
"%s:\n%s\n",
premiumListName,
Streams.stream(PremiumListDualDao.loadAllPremiumListEntries(premiumListName))
Streams.stream(PremiumListDao.loadAllPremiumListEntries(premiumListName))
.sorted(Comparator.comparing(PremiumListEntry::getLabel))
.map(PremiumListEntry::toString)
.collect(Collectors.joining("\n")));
core/src/main/java/google/registry/tools/RegistryTool.java
-1
View File
@@ -38,7 +38,6 @@ public final class RegistryTool {
.put("canonicalize_labels", CanonicalizeLabelsCommand.class)
.put("check_domain", CheckDomainCommand.class)
.put("check_domain_claims", CheckDomainClaimsCommand.class)
.put("compare_premium_lists", ComparePremiumListsCommand.class)
.put("compare_reserved_lists", CompareReservedListsCommand.class)
.put("convert_idn", ConvertIdnCommand.class)
.put("count_domains", CountDomainsCommand.class)
core/src/main/java/google/registry/tools/UpdatePremiumListCommand.java
+3 -3
View File
@@ -29,7 +29,7 @@ import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
import google.registry.persistence.VKey;
import google.registry.schema.tld.PremiumEntry;
import google.registry.schema.tld.PremiumListSqlDao;
import google.registry.schema.tld.PremiumListDao;
import google.registry.schema.tld.PremiumListUtils;
import java.nio.file.Files;
import java.util.List;
@@ -79,12 +79,12 @@ class UpdatePremiumListCommand extends CreateOrUpdatePremiumListCommand {
assertThat(persistedList.size()).isEqualTo(1);
*/
protected ImmutableSet<String> getExistingPremiumListEntry(String name) {
Optional<PremiumList> list = PremiumListSqlDao.getLatestRevision(name);
Optional<PremiumList> list = PremiumListDao.getLatestRevision(name);
checkArgument(
list.isPresent(),
String.format("Could not update premium list %s because it doesn't exist.", name));
Iterable<PremiumEntry> sqlListEntries =
jpaTm().transact(() -> PremiumListSqlDao.loadPremiumListEntriesUncached(list.get()));
jpaTm().transact(() -> PremiumListDao.loadPremiumListEntries(list.get()));
return Streams.stream(sqlListEntries)
.map(
premiumEntry ->
core/src/main/java/google/registry/tools/server/CreatePremiumListAction.java
+5 -3
View File
@@ -21,10 +21,10 @@ import static google.registry.request.Action.Method.POST;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.common.flogger.FluentLogger;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.auth.Auth;
import google.registry.schema.tld.PremiumListDao;
import java.util.List;
import javax.inject.Inject;
@@ -51,7 +51,9 @@ public class CreatePremiumListAction extends CreateOrUpdatePremiumListAction {
@Override
protected void save() {
checkArgument(
!PremiumListDualDao.exists(name), "A premium list of this name already exists: %s", name);
!PremiumListDao.getLatestRevision(name).isPresent(),
"A premium list of this name already exists: %s",
name);
if (!override) {
assertTldExists(
name,
@@ -62,7 +64,7 @@ public class CreatePremiumListAction extends CreateOrUpdatePremiumListAction {
logInputData();
List<String> inputDataPreProcessed =
Splitter.on('\n').omitEmptyStrings().splitToList(inputData);
PremiumListDualDao.save(name, inputDataPreProcessed);
PremiumListDao.save(name, inputDataPreProcessed);
String message =
String.format("Saved premium list %s with %d entries", name, inputDataPreProcessed.size());
logger.atInfo().log(message);
core/src/main/java/google/registry/tools/server/ListPremiumListsAction.java
+2 -2
View File
@@ -21,9 +21,9 @@ import static google.registry.request.Action.Method.POST;
import com.google.common.collect.ImmutableSet;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.request.Action;
import google.registry.request.auth.Auth;
import google.registry.schema.tld.PremiumListDao;
import java.util.Comparator;
import java.util.Optional;
import javax.inject.Inject;
@@ -55,7 +55,7 @@ public final class ListPremiumListsAction extends ListObjectsAction<PremiumList>
() ->
jpaTm().loadAllOf(PremiumList.class).stream()
.map(PremiumList::getName)
.map(PremiumListDualDao::getLatestRevision)
.map(PremiumListDao::getLatestRevision)
.filter(Optional::isPresent)
.map(Optional::get)
.peek(list -> Hibernate.initialize(list.getLabelsToPrices()))
core/src/main/java/google/registry/tools/server/UpdatePremiumListAction.java
+3 -3
View File
@@ -21,9 +21,9 @@ import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.common.flogger.FluentLogger;
import google.registry.model.registry.label.PremiumList;
import google.registry.model.registry.label.PremiumListDualDao;
import google.registry.request.Action;
import google.registry.request.auth.Auth;
import google.registry.schema.tld.PremiumListDao;
import java.util.List;
import javax.inject.Inject;
@@ -47,7 +47,7 @@ public class UpdatePremiumListAction extends CreateOrUpdatePremiumListAction {
@Override
protected void save() {
checkArgument(
PremiumListDualDao.exists(name),
PremiumListDao.getLatestRevision(name).isPresent(),
"Could not update premium list %s because it doesn't exist.",
name);
@@ -55,7 +55,7 @@ public class UpdatePremiumListAction extends CreateOrUpdatePremiumListAction {
logInputData();
List<String> inputDataPreProcessed =
Splitter.on('\n').omitEmptyStrings().splitToList(inputData);
PremiumList newPremiumList = PremiumListDualDao.save(name, inputDataPreProcessed);
PremiumList newPremiumList = PremiumListDao.save(name, inputDataPreProcessed);
String message =
String.format(