mirror of
https://github.com/google/nomulus
synced 2026-08-31 21:27:09 +00:00
Merge two reserved list entities (#616)
* Merge reserved list * Replace INSTANCE with getInstance() * Fix broken test * Rebase on master * Simplify class
This commit is contained in:
Shicong Huang
@@ -15,6 +15,7 @@
|
||||
package google.registry.model.registry.label;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
@@ -30,6 +31,7 @@ import com.google.common.collect.Multiset;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Ignore;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.Buildable;
|
||||
import google.registry.model.ImmutableObject;
|
||||
@@ -42,6 +44,11 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.MappedSuperclass;
|
||||
import javax.persistence.Transient;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
@@ -49,25 +56,46 @@ import org.joda.time.DateTime;
|
||||
*
|
||||
* @param <T> The type of the root value being listed, e.g. {@link ReservationType}.
|
||||
* @param <R> The type of domain label entry being listed, e.g. {@link ReservedListEntry} (note,
|
||||
* must subclass {@link DomainLabelEntry}.
|
||||
* must subclass {@link DomainLabelEntry}.
|
||||
*/
|
||||
@MappedSuperclass
|
||||
public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends DomainLabelEntry<T, ?>>
|
||||
extends ImmutableObject implements Buildable {
|
||||
|
||||
@Ignore
|
||||
@javax.persistence.Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
Long revisionId;
|
||||
|
||||
@Id
|
||||
@Column(nullable = false)
|
||||
String name;
|
||||
|
||||
@Parent
|
||||
Key<EntityGroupRoot> parent = getCrossTldKey();
|
||||
@Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
|
||||
|
||||
DateTime creationTime;
|
||||
@Transient DateTime creationTime;
|
||||
|
||||
// The list in Cloud SQL is immutable, we only have a creation_timestamp field and it should be
|
||||
// set to the timestamp when the list is created. In Datastore, we have two fields and the
|
||||
// lastUpdateTime is set to the current timestamp when creating and updating a list. So, we use
|
||||
// lastUpdateTime as the creation_timestamp column during the dual-write phase for compatibility.
|
||||
@Column(name = "creation_timestamp", nullable = false)
|
||||
DateTime lastUpdateTime;
|
||||
|
||||
/** Returns the ID of this revision, or throws if null. */
|
||||
public long getRevisionId() {
|
||||
checkState(
|
||||
revisionId != null,
|
||||
"revisionId is null because this object has not been persisted to the database yet");
|
||||
return revisionId;
|
||||
}
|
||||
|
||||
/** Returns the name of the reserved list. */
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
/** Returns the creation time of this revision of the reserved list. */
|
||||
public DateTime getCreationTime() {
|
||||
return creationTime;
|
||||
}
|
||||
@@ -183,6 +211,9 @@ public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends Dom
|
||||
@Override
|
||||
public T build() {
|
||||
checkArgument(!isNullOrEmpty(getInstance().name), "List must have a name");
|
||||
// The list is immutable in Cloud SQL, so make sure the revision id is not set when the
|
||||
// builder object is created from a list object
|
||||
getInstance().revisionId = null;
|
||||
return super.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,16 +23,20 @@ import com.google.common.net.InternetDomainName;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import google.registry.model.Buildable.GenericBuilder;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.MappedSuperclass;
|
||||
|
||||
/**
|
||||
* Represents a label entry parsed from a line in a reserved/premium list txt file.
|
||||
*
|
||||
* @param <T> The type of the value stored for the domain label, e.g. {@link ReservationType}.
|
||||
*/
|
||||
@MappedSuperclass
|
||||
public abstract class DomainLabelEntry<T extends Comparable<?>, D extends DomainLabelEntry<?, ?>>
|
||||
extends ImmutableObject implements Comparable<D> {
|
||||
|
||||
@Id
|
||||
@Column(name = "domain_label", insertable = false, updatable = false)
|
||||
String label;
|
||||
|
||||
String comment;
|
||||
|
||||
@@ -16,11 +16,8 @@ package google.registry.model.registry.label;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.collect.ImmutableMap.toImmutableMap;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
|
||||
import static google.registry.util.CollectionUtils.nullToEmpty;
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
@@ -30,12 +27,8 @@ import com.google.common.base.Splitter;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.MapDifference;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Embed;
|
||||
@@ -45,45 +38,58 @@ import com.googlecode.objectify.mapper.Mapper;
|
||||
import google.registry.model.Buildable;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.label.DomainLabelMetrics.MetricsReservedListMatch;
|
||||
import google.registry.schema.replay.DatastoreEntity;
|
||||
import google.registry.schema.replay.SqlEntity;
|
||||
import google.registry.schema.tld.ReservedList.ReservedEntry;
|
||||
import google.registry.schema.tld.ReservedListDao;
|
||||
import google.registry.schema.replay.DatastoreAndSqlEntity;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.persistence.CollectionTable;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.ElementCollection;
|
||||
import javax.persistence.Embeddable;
|
||||
import javax.persistence.Index;
|
||||
import javax.persistence.JoinColumn;
|
||||
import javax.persistence.MapKeyColumn;
|
||||
import javax.persistence.Table;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* A reserved list entity, persisted to Datastore, that is used to check domain label reservations.
|
||||
* A list of reserved domain labels that are blocked from being registered for various reasons.
|
||||
*
|
||||
* <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
|
||||
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly
|
||||
* succeeds, we will end up with having two exact same reserved lists that differ only by
|
||||
* revisionId. This is fine though, because we only use the list with the highest revisionId.
|
||||
*/
|
||||
@Entity
|
||||
@javax.persistence.Entity
|
||||
@Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")})
|
||||
public final class ReservedList
|
||||
extends BaseDomainLabelList<ReservationType, ReservedList.ReservedListEntry>
|
||||
implements DatastoreEntity {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
implements DatastoreAndSqlEntity {
|
||||
|
||||
@Mapify(ReservedListEntry.LabelMapper.class)
|
||||
@ElementCollection
|
||||
@CollectionTable(
|
||||
name = "ReservedEntry",
|
||||
joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
|
||||
@MapKeyColumn(name = "domain_label")
|
||||
Map<String, ReservedListEntry> reservedListMap;
|
||||
|
||||
@Column(nullable = false)
|
||||
boolean shouldPublish = true;
|
||||
|
||||
@Override
|
||||
public ImmutableList<SqlEntity> toSqlEntities() {
|
||||
return ImmutableList.of(); // ReservedList is dual-written
|
||||
}
|
||||
|
||||
/**
|
||||
* A reserved list entry entity, persisted to Datastore, that represents a single label and its
|
||||
* reservation type.
|
||||
*/
|
||||
@Embed
|
||||
public static class ReservedListEntry
|
||||
extends DomainLabelEntry<ReservationType, ReservedListEntry> implements Buildable {
|
||||
@Embeddable
|
||||
public static class ReservedListEntry extends DomainLabelEntry<ReservationType, ReservedListEntry>
|
||||
implements Buildable {
|
||||
|
||||
@Column(nullable = false)
|
||||
ReservationType reservationType;
|
||||
|
||||
/** Mapper for use with @Mapify */
|
||||
@@ -150,6 +156,7 @@ public final class ReservedList
|
||||
return shouldPublish;
|
||||
}
|
||||
|
||||
/** Returns a {@link Map} of domain labels to {@link ReservedListEntry}. */
|
||||
public ImmutableMap<String, ReservedListEntry> getReservedListEntries() {
|
||||
return ImmutableMap.copyOf(nullToEmpty(reservedListMap));
|
||||
}
|
||||
@@ -239,65 +246,10 @@ public final class ReservedList
|
||||
new CacheLoader<String, ReservedList>() {
|
||||
@Override
|
||||
public ReservedList load(String listName) {
|
||||
ReservedList datastoreList =
|
||||
ofy()
|
||||
.load()
|
||||
.type(ReservedList.class)
|
||||
.parent(getCrossTldKey())
|
||||
.id(listName)
|
||||
.now();
|
||||
// Also load the list from Cloud SQL, compare the two lists, and log if different.
|
||||
try {
|
||||
loadAndCompareCloudSqlList(datastoreList);
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().withCause(t).log("Error comparing reserved lists.");
|
||||
}
|
||||
return datastoreList;
|
||||
return ReservedListDualWriteDao.getLatestRevision(listName).orElse(null);
|
||||
}
|
||||
});
|
||||
|
||||
private static final void loadAndCompareCloudSqlList(ReservedList datastoreList) {
|
||||
Optional<google.registry.schema.tld.ReservedList> maybeCloudSqlList =
|
||||
ReservedListDao.getLatestRevision(datastoreList.getName());
|
||||
if (maybeCloudSqlList.isPresent()) {
|
||||
Map<String, ReservedEntry> datastoreLabelsToReservations =
|
||||
datastoreList.reservedListMap.entrySet().parallelStream()
|
||||
.collect(
|
||||
toImmutableMap(
|
||||
Map.Entry::getKey,
|
||||
entry ->
|
||||
ReservedEntry.create(
|
||||
entry.getValue().reservationType, entry.getValue().comment)));
|
||||
|
||||
google.registry.schema.tld.ReservedList cloudSqlList = maybeCloudSqlList.get();
|
||||
MapDifference<String, ReservedEntry> diff =
|
||||
Maps.difference(datastoreLabelsToReservations, cloudSqlList.getLabelsToReservations());
|
||||
if (!diff.areEqual()) {
|
||||
if (diff.entriesDiffering().size() > 10) {
|
||||
logger.atWarning().log(
|
||||
String.format(
|
||||
"Unequal reserved lists detected, Cloud SQL list with revision"
|
||||
+ " id %d has %d different records than the current"
|
||||
+ " Datastore list.",
|
||||
cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
|
||||
} else {
|
||||
StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
|
||||
diff.entriesDiffering()
|
||||
.forEach(
|
||||
(label, valueDiff) ->
|
||||
diffMessage.append(
|
||||
String.format(
|
||||
"Domain label %s has entry %s in Datastore and entry"
|
||||
+ " %s in Cloud SQL.\n",
|
||||
label, valueDiff.leftValue(), valueDiff.rightValue())));
|
||||
logger.atWarning().log(diffMessage.toString());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.atWarning().log("Reserved list in Cloud SQL is empty.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the {@link ReservationType} of a label in a single ReservedList, or returns an absent
|
||||
* Optional if none exists in the list.
|
||||
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.registry.label;
|
||||
|
||||
import static com.google.common.collect.ImmutableMap.toImmutableMap;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
|
||||
import com.google.common.collect.MapDifference;
|
||||
import com.google.common.collect.MapDifference.ValueDifference;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.registry.label.ReservedList.ReservedListEntry;
|
||||
import google.registry.persistence.VKey;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* A {@link ReservedList} DAO that does dual-write and dual-read against Datastore and Cloud SQL. It
|
||||
* still uses Datastore as the primary storage and suppresses any exception thrown by Cloud SQL.
|
||||
*
|
||||
* <p>TODO(b/160993806): Delete this DAO and switch to use the SQL only DAO after migrating to Cloud
|
||||
* SQL.
|
||||
*/
|
||||
public class ReservedListDualWriteDao {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private ReservedListDualWriteDao() {}
|
||||
|
||||
/** Persist a new reserved list to Cloud SQL. */
|
||||
public static void save(ReservedList reservedList) {
|
||||
ofyTm().transact(() -> ofyTm().saveNewOrUpdate(reservedList));
|
||||
try {
|
||||
logger.atInfo().log("Saving reserved list %s to Cloud SQL", reservedList.getName());
|
||||
ReservedListSqlDao.save(reservedList);
|
||||
logger.atInfo().log(
|
||||
"Saved reserved list %s with %d entries to Cloud SQL",
|
||||
reservedList.getName(), reservedList.getReservedListEntries().size());
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().withCause(t).log("Error saving the reserved list to Cloud SQL.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent revision of the {@link ReservedList} with the specified name, if it
|
||||
* exists.
|
||||
*/
|
||||
public static Optional<ReservedList> getLatestRevision(String reservedListName) {
|
||||
Optional<ReservedList> maybeDatastoreList =
|
||||
ofyTm()
|
||||
.maybeLoad(
|
||||
VKey.createOfy(
|
||||
ReservedList.class,
|
||||
Key.create(getCrossTldKey(), ReservedList.class, reservedListName)));
|
||||
try {
|
||||
// Also load the list from Cloud SQL, compare the two lists, and log if different.
|
||||
maybeDatastoreList.ifPresent(ReservedListDualWriteDao::loadAndCompareCloudSqlList);
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().withCause(t).log("Error comparing reserved lists.");
|
||||
}
|
||||
return maybeDatastoreList;
|
||||
}
|
||||
|
||||
private static void loadAndCompareCloudSqlList(ReservedList datastoreList) {
|
||||
Optional<ReservedList> maybeCloudSqlList =
|
||||
ReservedListSqlDao.getLatestRevision(datastoreList.getName());
|
||||
if (maybeCloudSqlList.isPresent()) {
|
||||
Map<String, ReservedListEntry> datastoreLabelsToReservations =
|
||||
datastoreList.reservedListMap.entrySet().parallelStream()
|
||||
.collect(
|
||||
toImmutableMap(
|
||||
Map.Entry::getKey,
|
||||
entry ->
|
||||
ReservedListEntry.create(
|
||||
entry.getKey(),
|
||||
entry.getValue().reservationType,
|
||||
entry.getValue().comment)));
|
||||
|
||||
ReservedList cloudSqlList = maybeCloudSqlList.get();
|
||||
MapDifference<String, ReservedListEntry> diff =
|
||||
Maps.difference(datastoreLabelsToReservations, cloudSqlList.reservedListMap);
|
||||
if (!diff.areEqual()) {
|
||||
if (diff.entriesDiffering().size() > 10) {
|
||||
logger.atWarning().log(
|
||||
String.format(
|
||||
"Unequal reserved lists detected, Cloud SQL list with revision"
|
||||
+ " id %d has %d different records than the current"
|
||||
+ " Datastore list.",
|
||||
cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
|
||||
} else {
|
||||
StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
|
||||
diff.entriesDiffering().entrySet().stream()
|
||||
.forEach(
|
||||
entry -> {
|
||||
String label = entry.getKey();
|
||||
ValueDifference<ReservedListEntry> valueDiff = entry.getValue();
|
||||
diffMessage.append(
|
||||
String.format(
|
||||
"Domain label %s has entry %s in Datastore and entry"
|
||||
+ " %s in Cloud SQL.\n",
|
||||
label, valueDiff.leftValue(), valueDiff.rightValue()));
|
||||
});
|
||||
logger.atWarning().log(diffMessage.toString());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.atWarning().log("Reserved list in Cloud SQL is empty.");
|
||||
}
|
||||
}
|
||||
}
|
||||
+32
-41
@@ -12,20 +12,46 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.schema.tld;
|
||||
package google.registry.model.registry.label;
|
||||
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
/** Data access object class for {@link ReservedList} */
|
||||
public class ReservedListDao {
|
||||
/**
|
||||
* A {@link ReservedList} DAO for Cloud SQL.
|
||||
*
|
||||
* <p>TODO(b/160993806): Rename this class to ReservedListDao after migrating to Cloud SQL.
|
||||
*/
|
||||
public class ReservedListSqlDao {
|
||||
|
||||
private ReservedListSqlDao() {}
|
||||
|
||||
/** Persist a new reserved list to Cloud SQL. */
|
||||
public static void save(ReservedList reservedList) {
|
||||
jpaTm().transact(() -> jpaTm().getEntityManager().persist(reservedList));
|
||||
checkArgumentNotNull(reservedList, "Must specify reservedList");
|
||||
jpaTm().transact(() -> jpaTm().saveNew(reservedList));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent revision of the {@link ReservedList} with the specified name, if it
|
||||
* exists.
|
||||
*/
|
||||
public static Optional<ReservedList> getLatestRevision(String reservedListName) {
|
||||
return jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm()
|
||||
.getEntityManager()
|
||||
.createQuery(
|
||||
"FROM ReservedList rl LEFT JOIN FETCH rl.reservedListMap WHERE"
|
||||
+ " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl"
|
||||
+ " WHERE subrl.name = :name)",
|
||||
ReservedList.class)
|
||||
.setParameter("name", reservedListName)
|
||||
.getResultStream()
|
||||
.findFirst());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -46,39 +72,4 @@ public class ReservedListDao {
|
||||
.size()
|
||||
> 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent revision of the {@link ReservedList} with the specified name, if it
|
||||
* exists. TODO(shicong): Change this method to package level access after dual-read phase.
|
||||
*/
|
||||
public static Optional<ReservedList> getLatestRevision(String reservedListName) {
|
||||
return jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm()
|
||||
.getEntityManager()
|
||||
.createQuery(
|
||||
"FROM ReservedList rl LEFT JOIN FETCH rl.labelsToReservations WHERE"
|
||||
+ " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl"
|
||||
+ " WHERE subrl.name = :name)",
|
||||
ReservedList.class)
|
||||
.setParameter("name", reservedListName)
|
||||
.getResultStream()
|
||||
.findFirst());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent revision of the {@link ReservedList} with the specified name, from
|
||||
* cache.
|
||||
*/
|
||||
public static Optional<ReservedList> getLatestRevisionCached(String reservedListName) {
|
||||
try {
|
||||
return ReservedListCache.cacheReservedLists.get(reservedListName);
|
||||
} catch (ExecutionException e) {
|
||||
throw new UncheckedExecutionException(
|
||||
"Could not retrieve reserved list named " + reservedListName, e);
|
||||
}
|
||||
}
|
||||
|
||||
private ReservedListDao() {}
|
||||
}
|
||||
@@ -58,6 +58,31 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
return new VKey(kind, null, sqlKey);
|
||||
}
|
||||
|
||||
/** Creates a {@link VKey} which only contains the ofy primary key. */
|
||||
public static <T> VKey<T> createOfy(
|
||||
Class<? extends T> kind, com.googlecode.objectify.Key<? extends T> ofyKey) {
|
||||
checkArgumentNotNull(kind, "kind must not be null");
|
||||
checkArgumentNotNull(ofyKey, "ofyKey must not be null");
|
||||
return new VKey(kind, ofyKey, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link VKey} which only contains the ofy primary key by specifying the id of the
|
||||
* {@link Key}.
|
||||
*/
|
||||
public static <T> VKey<T> createOfy(Class<? extends T> kind, long id) {
|
||||
return createOfy(kind, Key.create(kind, id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link VKey} which only contains the ofy primary key by specifying the name of the
|
||||
* {@link Key}.
|
||||
*/
|
||||
public static <T> VKey<T> createOfy(Class<? extends T> kind, String name) {
|
||||
checkArgumentNotNull(kind, "name must not be null");
|
||||
return createOfy(kind, Key.create(kind, name));
|
||||
}
|
||||
|
||||
/** Creates a {@link VKey} which only contains both sql and ofy primary key. */
|
||||
public static <T> VKey<T> create(
|
||||
Class<? extends T> kind, Object sqlKey, com.googlecode.objectify.Key ofyKey) {
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.schema.tld;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableList.sortedCopyOf;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.model.CreateAutoTimestamp;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.registry.label.ReservationType;
|
||||
import google.registry.schema.replay.DatastoreEntity;
|
||||
import google.registry.schema.replay.SqlEntity;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.persistence.CollectionTable;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.ElementCollection;
|
||||
import javax.persistence.Embeddable;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Index;
|
||||
import javax.persistence.JoinColumn;
|
||||
import javax.persistence.MapKeyColumn;
|
||||
import javax.persistence.Table;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* A list of reserved domain labels that are blocked from being registered for various reasons.
|
||||
*
|
||||
* <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
|
||||
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly
|
||||
* succeeds, we will end up with having two exact same reserved lists that differ only by
|
||||
* revisionId. This is fine though, because we only use the list with the highest revisionId.
|
||||
*/
|
||||
@Entity
|
||||
@Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")})
|
||||
public class ReservedList extends ImmutableObject implements SqlEntity {
|
||||
|
||||
@Column(nullable = false)
|
||||
private String name;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
@Column(nullable = false)
|
||||
private Long revisionId;
|
||||
|
||||
@Column(nullable = false)
|
||||
private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
|
||||
|
||||
@Column(nullable = false)
|
||||
private Boolean shouldPublish;
|
||||
|
||||
@ElementCollection
|
||||
@CollectionTable(
|
||||
name = "ReservedEntry",
|
||||
joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
|
||||
@MapKeyColumn(name = "domainLabel")
|
||||
private Map<String, ReservedEntry> labelsToReservations;
|
||||
|
||||
@Override
|
||||
public ImmutableList<DatastoreEntity> toDatastoreEntities() {
|
||||
return ImmutableList.of(); // ReservedList is dual-written\
|
||||
}
|
||||
|
||||
@Embeddable
|
||||
public static class ReservedEntry extends ImmutableObject {
|
||||
@Column(nullable = false)
|
||||
private ReservationType reservationType;
|
||||
|
||||
@Column(nullable = true)
|
||||
private String comment;
|
||||
|
||||
private ReservedEntry(ReservationType reservationType, @Nullable String comment) {
|
||||
this.reservationType = reservationType;
|
||||
this.comment = comment;
|
||||
}
|
||||
|
||||
// Hibernate requires this default constructor.
|
||||
private ReservedEntry() {}
|
||||
|
||||
/** Constructs a {@link ReservedEntry} object. */
|
||||
public static ReservedEntry create(ReservationType reservationType, @Nullable String comment) {
|
||||
return new ReservedEntry(reservationType, comment);
|
||||
}
|
||||
|
||||
/** Returns the reservation type for this entry. */
|
||||
public ReservationType getReservationType() {
|
||||
return reservationType;
|
||||
}
|
||||
|
||||
/** Returns the comment for this entry. Retruns null if there is no comment. */
|
||||
public String getComment() {
|
||||
return comment;
|
||||
}
|
||||
}
|
||||
|
||||
private ReservedList(
|
||||
String name, Boolean shouldPublish, Map<String, ReservedEntry> labelsToReservations) {
|
||||
this.name = name;
|
||||
this.shouldPublish = shouldPublish;
|
||||
this.labelsToReservations = labelsToReservations;
|
||||
}
|
||||
|
||||
// Hibernate requires this default constructor.
|
||||
private ReservedList() {}
|
||||
|
||||
/** Constructs a {@link ReservedList} object. */
|
||||
public static ReservedList create(
|
||||
String name, Boolean shouldPublish, Map<String, ReservedEntry> labelsToReservations) {
|
||||
ImmutableList<String> invalidLabels =
|
||||
labelsToReservations.entrySet().parallelStream()
|
||||
.flatMap(
|
||||
entry -> {
|
||||
String label = entry.getKey();
|
||||
if (label.equals(canonicalizeDomainName(label))) {
|
||||
return Stream.empty();
|
||||
} else {
|
||||
return Stream.of(label);
|
||||
}
|
||||
})
|
||||
.collect(toImmutableList());
|
||||
checkArgument(
|
||||
invalidLabels.isEmpty(),
|
||||
"Label(s) [%s] must be in puny-coded, lower-case form",
|
||||
Joiner.on(",").join(sortedCopyOf(invalidLabels)));
|
||||
return new ReservedList(name, shouldPublish, labelsToReservations);
|
||||
}
|
||||
|
||||
/** Returns the name of the reserved list. */
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
/** Returns the ID of this revision, or throws if null. */
|
||||
public Long getRevisionId() {
|
||||
checkState(
|
||||
revisionId != null,
|
||||
"revisionId is null because this object has not been persisted to the database yet");
|
||||
return revisionId;
|
||||
}
|
||||
|
||||
/** Returns the creation time of this revision of the reserved list. */
|
||||
public DateTime getCreationTimestamp() {
|
||||
return creationTimestamp.getTimestamp();
|
||||
}
|
||||
|
||||
/** Returns a {@link Map} of domain labels to {@link ReservedEntry}. */
|
||||
public ImmutableMap<String, ReservedEntry> getLabelsToReservations() {
|
||||
return ImmutableMap.copyOf(labelsToReservations);
|
||||
}
|
||||
|
||||
/** Returns true if the reserved list should be published. */
|
||||
public Boolean getShouldPublish() {
|
||||
return shouldPublish;
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.schema.tld;
|
||||
|
||||
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.util.Optional;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/** Caching utils for {@link ReservedList} */
|
||||
public class ReservedListCache {
|
||||
|
||||
/**
|
||||
* In-memory cache for reserved lists.
|
||||
*
|
||||
* <p>This is cached for a shorter duration because we need to periodically reload from the DB to
|
||||
* check if a new revision has been published, and if so, then use that.
|
||||
*/
|
||||
@NonFinalForTesting
|
||||
static LoadingCache<String, Optional<ReservedList>> cacheReservedLists =
|
||||
createCacheReservedLists(getDomainLabelListCacheDuration());
|
||||
|
||||
@VisibleForTesting
|
||||
static LoadingCache<String, Optional<ReservedList>> createCacheReservedLists(
|
||||
Duration cachePersistDuration) {
|
||||
return CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(cachePersistDuration.getMillis(), MILLISECONDS)
|
||||
.build(
|
||||
new CacheLoader<String, Optional<ReservedList>>() {
|
||||
@Override
|
||||
public Optional<ReservedList> load(String reservedListName) {
|
||||
return ReservedListDao.getLatestRevision(reservedListName);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -14,23 +14,12 @@
|
||||
|
||||
package google.registry.tools;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.model.registry.label.BaseDomainLabelList.splitOnComment;
|
||||
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
|
||||
|
||||
import com.beust.jcommander.Parameter;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.HashMultiset;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.collect.Multiset;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.model.registry.label.ReservationType;
|
||||
import google.registry.schema.tld.ReservedList.ReservedEntry;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.registry.label.ReservedListDualWriteDao;
|
||||
import google.registry.tools.params.PathParameter;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
/**
|
||||
@@ -62,69 +51,22 @@ public abstract class CreateOrUpdateReservedListCommand extends MutatingCommand
|
||||
arity = 1)
|
||||
Boolean shouldPublish;
|
||||
|
||||
google.registry.schema.tld.ReservedList cloudSqlReservedList;
|
||||
|
||||
abstract void saveToCloudSql();
|
||||
ReservedList reservedList;
|
||||
|
||||
@Override
|
||||
protected String execute() throws Exception {
|
||||
// Save the list to Datastore and output its response.
|
||||
String output = super.execute();
|
||||
logger.atInfo().log(output);
|
||||
|
||||
String cloudSqlMessage =
|
||||
protected String execute() {
|
||||
String message =
|
||||
String.format(
|
||||
"Saved reserved list %s with %d entries",
|
||||
name, cloudSqlReservedList.getLabelsToReservations().size());
|
||||
name, reservedList.getReservedListEntries().size());
|
||||
try {
|
||||
logger.atInfo().log("Saving reserved list to Cloud SQL for TLD %s", name);
|
||||
saveToCloudSql();
|
||||
logger.atInfo().log(cloudSqlMessage);
|
||||
logger.atInfo().log("Saving reserved list for TLD %s", name);
|
||||
ReservedListDualWriteDao.save(reservedList);
|
||||
logger.atInfo().log(message);
|
||||
} catch (Throwable e) {
|
||||
cloudSqlMessage =
|
||||
"Unexpected error saving reserved list to Cloud SQL from nomulus tool command";
|
||||
logger.atSevere().withCause(e).log(cloudSqlMessage);
|
||||
message = "Unexpected error saving reserved list from nomulus tool command";
|
||||
logger.atSevere().withCause(e).log(message);
|
||||
}
|
||||
return cloudSqlMessage;
|
||||
}
|
||||
|
||||
/** Turns the list CSV data into a map of labels to {@link ReservedEntry}. */
|
||||
static ImmutableMap<String, ReservedEntry> parseToReservationsByLabels(Iterable<String> lines) {
|
||||
Map<String, ReservedEntry> labelsToEntries = Maps.newHashMap();
|
||||
Multiset<String> duplicateLabels = HashMultiset.create();
|
||||
for (String originalLine : lines) {
|
||||
List<String> lineAndComment = splitOnComment(originalLine);
|
||||
if (lineAndComment.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
String line = lineAndComment.get(0);
|
||||
String comment = lineAndComment.get(1);
|
||||
List<String> parts = Splitter.on(',').trimResults().splitToList(line);
|
||||
checkArgument(
|
||||
parts.size() == 2 || parts.size() == 3,
|
||||
"Could not parse line in reserved list: %s",
|
||||
originalLine);
|
||||
String label = parts.get(0);
|
||||
checkArgument(
|
||||
label.equals(canonicalizeDomainName(label)),
|
||||
"Label '%s' must be in puny-coded, lower-case form",
|
||||
label);
|
||||
ReservationType reservationType = ReservationType.valueOf(parts.get(1));
|
||||
ReservedEntry reservedEntry = ReservedEntry.create(reservationType, comment);
|
||||
// Check if the label was already processed for this list (which is an error), and if so,
|
||||
// accumulate it so that a list of all duplicates can be thrown.
|
||||
if (labelsToEntries.containsKey(label)) {
|
||||
duplicateLabels.add(label, duplicateLabels.contains(label) ? 1 : 2);
|
||||
} else {
|
||||
labelsToEntries.put(label, reservedEntry);
|
||||
}
|
||||
}
|
||||
if (!duplicateLabels.isEmpty()) {
|
||||
throw new IllegalStateException(
|
||||
String.format(
|
||||
"Reserved list cannot contain duplicate labels. Dupes (with counts) were: %s",
|
||||
duplicateLabels));
|
||||
}
|
||||
return ImmutableMap.copyOf(labelsToEntries);
|
||||
return message;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ package google.registry.tools;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.model.registry.Registries.assertTldExists;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.ListNamingUtils.convertFilePathToName;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
@@ -27,7 +26,6 @@ import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.base.Strings;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.schema.tld.ReservedListDao;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -50,15 +48,14 @@ final class CreateReservedListCommand extends CreateOrUpdateReservedListCommand
|
||||
protected void init() throws Exception {
|
||||
name = Strings.isNullOrEmpty(name) ? convertFilePathToName(input) : name;
|
||||
checkArgument(
|
||||
!ReservedList.get(name).isPresent(),
|
||||
"A reserved list already exists by this name");
|
||||
!ReservedList.get(name).isPresent(), "A reserved list already exists by this name");
|
||||
if (!override) {
|
||||
validateListName(name);
|
||||
}
|
||||
DateTime now = DateTime.now(UTC);
|
||||
List<String> allLines = Files.readAllLines(input, UTF_8);
|
||||
boolean shouldPublish = this.shouldPublish == null || this.shouldPublish;
|
||||
ReservedList reservedList =
|
||||
reservedList =
|
||||
new ReservedList.Builder()
|
||||
.setName(name)
|
||||
.setReservedListMapFromLines(allLines)
|
||||
@@ -66,23 +63,6 @@ final class CreateReservedListCommand extends CreateOrUpdateReservedListCommand
|
||||
.setCreationTime(now)
|
||||
.setLastUpdateTime(now)
|
||||
.build();
|
||||
stageEntityChange(null, reservedList);
|
||||
cloudSqlReservedList =
|
||||
google.registry.schema.tld.ReservedList.create(
|
||||
name, shouldPublish, parseToReservationsByLabels(allLines));
|
||||
}
|
||||
|
||||
@Override
|
||||
void saveToCloudSql() {
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
checkArgument(
|
||||
!ReservedListDao.checkExists(cloudSqlReservedList.getName()),
|
||||
"A reserved list of this name already exists: %s.",
|
||||
cloudSqlReservedList.getName());
|
||||
ReservedListDao.save(cloudSqlReservedList);
|
||||
});
|
||||
}
|
||||
|
||||
private static void validateListName(String name) {
|
||||
|
||||
@@ -15,18 +15,17 @@
|
||||
package google.registry.tools;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.ListNamingUtils.convertFilePathToName;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.base.Strings;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.schema.tld.ReservedListDao;
|
||||
import google.registry.util.SystemClock;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Command to safely update {@link ReservedList} on Datastore. */
|
||||
@Parameters(separators = " =", commandDescription = "Update a ReservedList in Datastore.")
|
||||
@@ -35,42 +34,20 @@ final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand
|
||||
@Override
|
||||
protected void init() throws Exception {
|
||||
name = Strings.isNullOrEmpty(name) ? convertFilePathToName(input) : name;
|
||||
// TODO(shicong): Read existing entry from Cloud SQL
|
||||
Optional<ReservedList> existing = ReservedList.get(name);
|
||||
checkArgument(
|
||||
existing.isPresent(), "Could not update reserved list %s because it doesn't exist.", name);
|
||||
boolean shouldPublish =
|
||||
this.shouldPublish == null ? existing.get().getShouldPublish() : this.shouldPublish;
|
||||
List<String> allLines = Files.readAllLines(input, UTF_8);
|
||||
DateTime now = new SystemClock().nowUtc();
|
||||
ReservedList.Builder updated =
|
||||
existing
|
||||
.get()
|
||||
.asBuilder()
|
||||
.setReservedListMapFromLines(allLines)
|
||||
.setLastUpdateTime(new SystemClock().nowUtc())
|
||||
.setLastUpdateTime(now)
|
||||
.setShouldPublish(shouldPublish);
|
||||
stageEntityChange(existing.get(), updated.build());
|
||||
cloudSqlReservedList =
|
||||
google.registry.schema.tld.ReservedList.create(
|
||||
name, shouldPublish, parseToReservationsByLabels(allLines));
|
||||
}
|
||||
|
||||
@Override
|
||||
void saveToCloudSql() {
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
// This check is currently disabled because, during the Cloud SQL migration, we need
|
||||
// to be able to update reserved lists in Datastore while simultaneously creating
|
||||
// their first revision in Cloud SQL (i.e. if they haven't been migrated over yet).
|
||||
// TODO(shicong): Re-instate this once all reserved lists are migrated to Cloud SQL,
|
||||
// and add a unit test to verity that an exception will be thrown if
|
||||
// the reserved list doesn't exist.
|
||||
// checkArgument(
|
||||
// ReservedListDao.checkExists(cloudSqlReservedList.getName()),
|
||||
// "A reserved list of this name doesn't exist: %s.",
|
||||
// cloudSqlReservedList.getName());
|
||||
ReservedListDao.save(cloudSqlReservedList);
|
||||
});
|
||||
reservedList = updated.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,12 +36,12 @@
|
||||
<class>google.registry.schema.server.Lock</class>
|
||||
<class>google.registry.schema.tld.PremiumList</class>
|
||||
<class>google.registry.schema.tld.PremiumEntry</class>
|
||||
<class>google.registry.schema.tld.ReservedList</class>
|
||||
<class>google.registry.model.domain.secdns.DelegationSignerData</class>
|
||||
<class>google.registry.model.domain.GracePeriod</class>
|
||||
<class>google.registry.model.poll.PollMessage</class>
|
||||
<class>google.registry.model.poll.PollMessage$OneTime</class>
|
||||
<class>google.registry.model.poll.PollMessage$Autorenew</class>
|
||||
<class>google.registry.model.registry.label.ReservedList</class>
|
||||
|
||||
<!-- Customized type converters -->
|
||||
<class>google.registry.persistence.converter.BillingCostTransitionConverter</class>
|
||||
|
||||
Reference in New Issue
Block a user