mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-06-09 08:22:59 +00:00
Code Issues Releases Wiki Activity

Add initial support to write/update reserved lists to Cloud SQL (#388)

Browse Source 
* Add initial support to write reserved list to Cloud SQL

* Add support to update reserved list in Cloud SQL

* Fix wrong check when override is enabled in create command

* Add sql dependent tests to the suite

* Address comment

* Make invocation of super.execute more readable

* Add test to check upper case and non-puny code label

* Move ReservedListDao related classes to schema package

* Simplify a function name
This commit is contained in:
Shicong Huang
2019-12-19 12:51:48 -05:00
committed by GitHub
parent f301314d97
commit f76030f7f0
13 changed files with 573 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java
View File

@@ -121,7 +121,7 @@ public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends Dom
* (sans comment) and the comment (in that order). If the line was blank or empty, then this
* method returns an empty list.
*/
protected static List<String> splitOnComment(String line) {
public static List<String> splitOnComment(String line) {
String comment = "";
int index = line.indexOf('#');
if (index != -1) {

23
core/src/main/java/google/registry/schema/tld/ReservedList.java
View File

@@ -14,13 +14,20 @@
package google.registry.schema.tld;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.sortedCopyOf;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import google.registry.model.CreateAutoTimestamp;
import google.registry.model.ImmutableObject;
import google.registry.model.registry.label.ReservationType;
import java.util.Map;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
@@ -114,6 +121,22 @@ public class ReservedList extends ImmutableObject {
/** Constructs a {@link ReservedList} object. */
public static ReservedList create(
String name, Boolean shouldPublish, Map<String, ReservedEntry> labelsToReservations) {
ImmutableList<String> invalidLabels =
labelsToReservations.entrySet().parallelStream()
.flatMap(
entry -> {
String label = entry.getKey();
if (label.equals(canonicalizeDomainName(label))) {
return Stream.empty();
} else {
return Stream.of(label);
}
})
.collect(toImmutableList());
checkArgument(
invalidLabels.isEmpty(),
"Label(s) [%s] must be in puny-coded, lower-case form",
Joiner.on(",").join(sortedCopyOf(invalidLabels)));
return new ReservedList(name, shouldPublish, labelsToReservations);
}

47
core/src/main/java/google/registry/schema/tld/ReservedListDao.java Normal file
View File

@@ -0,0 +1,47 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.schema.tld;
import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
/** Data access object class for {@link ReservedList} */
public class ReservedListDao {
/** Persist a new reserved list to Cloud SQL. */
public static void save(ReservedList reservedList) {
jpaTm().transact(() -> jpaTm().getEntityManager().persist(reservedList));
}
/**
* Returns whether the reserved list of the given name exists.
*
* <p>This means that at least one reserved list revision must exist for the given name.
*/
public static boolean checkExists(String reservedListName) {
return jpaTm()
.transact(
() ->
jpaTm()
.getEntityManager()
.createQuery("SELECT 1 FROM ReservedList WHERE name = :name", Integer.class)
.setParameter("name", reservedListName)
.setMaxResults(1)
.getResultList()
.size()
> 0);
}
private ReservedListDao() {}
}

93
core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java
View File

@@ -14,9 +14,23 @@
package google.registry.tools;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.model.registry.label.BaseDomainLabelList.splitOnComment;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import com.beust.jcommander.Parameter;
import com.google.common.base.Splitter;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Multiset;
import com.google.common.flogger.FluentLogger;
import google.registry.model.registry.label.ReservationType;
import google.registry.schema.tld.ReservedList.ReservedEntry;
import google.registry.tools.params.PathParameter;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
/**
@@ -25,6 +39,8 @@ import javax.annotation.Nullable;
*/
public abstract class CreateOrUpdateReservedListCommand extends MutatingCommand {
static final FluentLogger logger = FluentLogger.forEnclosingClass();
@Nullable
@Parameter(
names = {"-n", "--name"},
@@ -45,4 +61,81 @@ public abstract class CreateOrUpdateReservedListCommand extends MutatingCommand
"Whether the list is published to the concatenated list on Drive (defaults to true).",
arity = 1)
Boolean shouldPublish;
@Parameter(
names = {"--also_cloud_sql"},
description =
"Persist reserved list to Cloud SQL in addition to Datastore; defaults to false.")
boolean alsoCloudSql;
google.registry.schema.tld.ReservedList cloudSqlReservedList;
abstract void saveToCloudSql();
@Override
protected String execute() throws Exception {
// Save the list to Datastore and output its response.
String output = super.execute();
logger.atInfo().log(output);
String cloudSqlMessage;
if (alsoCloudSql) {
cloudSqlMessage =
String.format(
"Saved reserved list %s with %d entries",
name, cloudSqlReservedList.getLabelsToReservations().size());
try {
logger.atInfo().log("Saving reserved list to Cloud SQL for TLD %s", name);
saveToCloudSql();
logger.atInfo().log(cloudSqlMessage);
} catch (Throwable e) {
cloudSqlMessage =
"Unexpected error saving reserved list to Cloud SQL from nomulus tool command";
logger.atSevere().withCause(e).log(cloudSqlMessage);
}
} else {
cloudSqlMessage = "Persisting reserved list to Cloud SQL is not enabled";
}
return cloudSqlMessage;
}
/** Turns the list CSV data into a map of labels to {@link ReservedEntry}. */
static ImmutableMap<String, ReservedEntry> parseToReservationsByLabels(Iterable<String> lines) {
Map<String, ReservedEntry> labelsToEntries = Maps.newHashMap();
Multiset<String> duplicateLabels = HashMultiset.create();
for (String originalLine : lines) {
List<String> lineAndComment = splitOnComment(originalLine);
if (lineAndComment.isEmpty()) {
continue;
}
String line = lineAndComment.get(0);
String comment = lineAndComment.get(1);
List<String> parts = Splitter.on(',').trimResults().splitToList(line);
checkArgument(
parts.size() == 2 || parts.size() == 3,
"Could not parse line in reserved list: %s",
originalLine);
String label = parts.get(0);
checkArgument(
label.equals(canonicalizeDomainName(label)),
"Label '%s' must be in puny-coded, lower-case form",
label);
ReservationType reservationType = ReservationType.valueOf(parts.get(1));
ReservedEntry reservedEntry = ReservedEntry.create(reservationType, comment);
// Check if the label was already processed for this list (which is an error), and if so,
// accumulate it so that a list of all duplicates can be thrown.
if (labelsToEntries.containsKey(label)) {
duplicateLabels.add(label, duplicateLabels.contains(label) ? 1 : 2);
} else {
labelsToEntries.put(label, reservedEntry);
}
}
if (!duplicateLabels.isEmpty()) {
throw new IllegalStateException(
String.format(
"Reserved list cannot contain duplicate labels. Dupes (with counts) were: %s",
duplicateLabels));
}
return ImmutableMap.copyOf(labelsToEntries);
}
}

26
core/src/main/java/google/registry/tools/CreateReservedListCommand.java
View File

@@ -16,6 +16,7 @@ package google.registry.tools;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.model.registry.Registries.assertTldExists;
import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.ListNamingUtils.convertFilePathToName;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.joda.time.DateTimeZone.UTC;
@@ -26,6 +27,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import google.registry.model.registry.label.ReservedList;
import google.registry.schema.tld.ReservedListDao;
import java.nio.file.Files;
import java.util.List;
import org.joda.time.DateTime;
@@ -54,15 +56,35 @@ final class CreateReservedListCommand extends CreateOrUpdateReservedListCommand
validateListName(name);
}
DateTime now = DateTime.now(UTC);
List<String> allLines = Files.readAllLines(input, UTF_8);
boolean shouldPublish = this.shouldPublish == null || this.shouldPublish;
ReservedList reservedList =
new ReservedList.Builder()
.setName(name)
.setReservedListMapFromLines(Files.readAllLines(input, UTF_8))
.setShouldPublish(shouldPublish == null || shouldPublish)
.setReservedListMapFromLines(allLines)
.setShouldPublish(shouldPublish)
.setCreationTime(now)
.setLastUpdateTime(now)
.build();
stageEntityChange(null, reservedList);
if (alsoCloudSql) {
cloudSqlReservedList =
google.registry.schema.tld.ReservedList.create(
name, shouldPublish, parseToReservationsByLabels(allLines));
}
}
@Override
void saveToCloudSql() {
jpaTm()
.transact(
() -> {
checkArgument(
!ReservedListDao.checkExists(cloudSqlReservedList.getName()),
"A reserved list of this name already exists: %s.",
cloudSqlReservedList.getName());
ReservedListDao.save(cloudSqlReservedList);
});
}
private static void validateListName(String name) {

33
core/src/main/java/google/registry/tools/UpdateReservedListCommand.java
View File

@@ -15,14 +15,17 @@
package google.registry.tools;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.ListNamingUtils.convertFilePathToName;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.beust.jcommander.Parameters;
import com.google.common.base.Strings;
import google.registry.model.registry.label.ReservedList;
import google.registry.schema.tld.ReservedListDao;
import google.registry.util.SystemClock;
import java.nio.file.Files;
import java.util.List;
import java.util.Optional;
/** Command to safely update {@link ReservedList} on Datastore. */
@@ -32,18 +35,38 @@ final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand
@Override
protected void init() throws Exception {
name = Strings.isNullOrEmpty(name) ? convertFilePathToName(input) : name;
// TODO(shicong): Read existing entry from Cloud SQL
Optional<ReservedList> existing = ReservedList.get(name);
checkArgument(
existing.isPresent(), "Could not update reserved list %s because it doesn't exist.", name);
boolean shouldPublish =
this.shouldPublish == null ? existing.get().getShouldPublish() : this.shouldPublish;
List<String> allLines = Files.readAllLines(input, UTF_8);
ReservedList.Builder updated =
existing
.get()
.asBuilder()
.setReservedListMapFromLines(Files.readAllLines(input, UTF_8))
.setLastUpdateTime(new SystemClock().nowUtc());
if (shouldPublish != null) {
updated.setShouldPublish(shouldPublish);
}
.setReservedListMapFromLines(allLines)
.setLastUpdateTime(new SystemClock().nowUtc())
.setShouldPublish(shouldPublish);
stageEntityChange(existing.get(), updated.build());
if (alsoCloudSql) {
cloudSqlReservedList =
google.registry.schema.tld.ReservedList.create(
name, shouldPublish, parseToReservationsByLabels(allLines));
}
}
@Override
void saveToCloudSql() {
jpaTm()
.transact(
() -> {
checkArgument(
ReservedListDao.checkExists(cloudSqlReservedList.getName()),
"A reserved list of this name doesn't exist: %s.",
cloudSqlReservedList.getName());
ReservedListDao.save(cloudSqlReservedList);
});
}
}