mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-01-06 21:47:31 +00:00
Code Issues Releases Wiki Activity

Change GenerateEscrowDepositCommand to trigger back end deposit generation

Browse Source 
Previously, GenerateEscrowDepositCommand generated the deposit itself. Channeling it through the existing deposit generation code make things more maintainable.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=152847950
This commit is contained in:
mountford
2017-04-11 12:53:07 -07:00
committed by Ben McIlwain
parent 8653d2f204
commit dea386d08a
7 changed files with 224 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
java/google/registry/rde/RdeModule.java
View File

@@ -40,11 +40,11 @@ import org.joda.time.DateTime;
@Module
public final class RdeModule {
static final String PARAM_WATERMARK = "watermark";
static final String PARAM_MANUAL = "manual";
static final String PARAM_DIRECTORY = "directory";
static final String PARAM_MODE = "mode";
static final String PARAM_REVISION = "revision";
public static final String PARAM_WATERMARK = "watermark";
public static final String PARAM_MANUAL = "manual";
public static final String PARAM_DIRECTORY = "directory";
public static final String PARAM_MODE = "mode";
public static final String PARAM_REVISION = "revision";
@Provides
@Parameter(PARAM_WATERMARK)

8
java/google/registry/rde/RdeStagingAction.java
View File

@@ -181,9 +181,11 @@ import org.joda.time.Duration;
* @see <a href="https://tools.ietf.org/html/draft-arias-noguchi-registry-data-escrow-06">Registry Data Escrow Specification</a>
* @see <a href="https://tools.ietf.org/html/draft-arias-noguchi-dnrd-objects-mapping-05">Domain Name Registration Data Objects Mapping</a>
*/
@Action(path = "/_dr/task/rdeStaging")
@Action(path = RdeStagingAction.PATH)
public final class RdeStagingAction implements Runnable {
public static final String PATH = "/_dr/task/rdeStaging";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject Clock clock;
@@ -302,6 +304,10 @@ public final class RdeStagingAction implements Runnable {
}
}
if (revision.isPresent() && (revision.get() < 0)) {
throw new BadRequestException("Revision must be greater than or equal to zero");
}
ImmutableSetMultimap.Builder<String, PendingDeposit> pendingsBuilder =
new ImmutableSetMultimap.Builder<>();

1
java/google/registry/tools/BUILD
View File

@@ -49,6 +49,7 @@ java_library(
"//java/google/registry/model",
"//java/google/registry/pricing",
"//java/google/registry/rde",
"//java/google/registry/request",
"//java/google/registry/request:modules",
"//java/google/registry/security",
"//java/google/registry/tldconfig/idn",

254
java/google/registry/tools/GenerateEscrowDepositCommand.java
View File

@@ -14,234 +14,102 @@
package google.registry.tools;
import static google.registry.model.EppResourceUtils.loadAtPointInTime;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.registry.Registries.assertTldExists;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static org.joda.time.DateTimeZone.UTC;
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import static google.registry.model.registry.Registries.assertTldsExist;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import com.beust.jcommander.Parameters;
import com.google.common.base.Function;
import com.google.common.base.Predicates;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Result;
import google.registry.config.RegistryConfig.Config;
import google.registry.model.EppResource;
import google.registry.model.EppResourceUtils;
import google.registry.model.ImmutableObject;
import google.registry.model.contact.ContactResource;
import google.registry.model.domain.DomainResource;
import google.registry.model.host.HostResource;
import google.registry.model.index.EppResourceIndex;
import google.registry.model.index.EppResourceIndexBucket;
import com.google.appengine.api.modules.ModulesService;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.common.base.Joiner;
import google.registry.model.rde.RdeMode;
import google.registry.model.rde.RdeNamingUtils;
import google.registry.model.registrar.Registrar;
import google.registry.rde.DepositFragment;
import google.registry.rde.RdeCounter;
import google.registry.rde.RdeMarshaller;
import google.registry.rde.RdeResourceType;
import google.registry.rde.RdeUtil;
import google.registry.tldconfig.idn.IdnTableEnum;
import google.registry.rde.RdeModule;
import google.registry.rde.RdeStagingAction;
import google.registry.request.RequestParameters;
import google.registry.tools.Command.RemoteApiCommand;
import google.registry.tools.params.DateTimeParameter;
import google.registry.tools.params.PathParameter;
import google.registry.xjc.rdeheader.XjcRdeHeader;
import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Named;
import org.joda.time.DateTime;
/** Command to generate an XML RDE escrow deposit (with relevant files) in current directory. */
/**
* Command to kick off the server-side generation of an XML RDE or BRDA escrow deposit, which will
* be stored in the specified manual subdirectory of the GCS RDE bucket.
*/
@Parameters(separators = " =", commandDescription = "Generate an XML escrow deposit.")
final class GenerateEscrowDepositCommand implements RemoteApiCommand {
@Parameter(
names = {"-t", "--tld"},
description = "Top level domain for which deposit should be generated.",
description = "Top level domain(s) for which deposit should be generated.",
required = true)
private String tld;
private List<String> tlds;
@Parameter(
names = {"-w", "--watermark"},
description = "Point-in-time timestamp for snapshotting Datastore.",
validateWith = DateTimeParameter.class)
private DateTime watermark = DateTime.now(UTC);
description = "Point-in-time timestamp(s) for snapshotting Datastore.",
required = true,
converter = DateTimeParameter.class)
private List<DateTime> watermarks;
@Parameter(
names = {"-m", "--mode"},
description = "FULL/THIN mode of operation.")
description = "Mode of operation: FULL for RDE deposits, THIN for BRDA deposits.")
private RdeMode mode = RdeMode.FULL;
@Parameter(
names = {"-r", "--revision"},
description = "Revision number. Use >0 for resends.")
private int revision = 0;
private Integer revision;
@Parameter(
names = {"-o", "--outdir"},
description = "Specify output directory. Default is current directory.",
validateWith = PathParameter.OutputDirectory.class)
private Path outdir = Paths.get(".");
description = "Specify output subdirectory (under GCS RDE bucket, directory manual).",
required = true)
private String outdir;
@Inject
EscrowDepositEncryptor encryptor;
@Inject
RdeCounter counter;
@Inject
@Config("eppResourceIndexBucketCount")
int eppResourceIndexBucketCount;
@Inject ModulesService modulesService;
@Inject @Named("rde-report") Queue queue;
@Override
public void run() throws Exception {
RdeMarshaller marshaller = new RdeMarshaller();
assertTldExists(tld);
String suffix = String.format("-%s-%s.tmp.xml", tld, watermark);
Path xmlPath = outdir.resolve("deposit" + suffix);
Path reportPath = outdir.resolve("report" + suffix);
try {
String id = RdeUtil.timestampToId(watermark);
XjcRdeHeader header;
try (OutputStream xmlOutputBytes = Files.newOutputStream(xmlPath);
Writer xmlOutput = new OutputStreamWriter(xmlOutputBytes, UTF_8)) {
xmlOutput.write(
marshaller.makeHeader(id, watermark, RdeResourceType.getUris(mode), revision));
for (ImmutableObject resource
: Iterables.concat(Registrar.loadAll(), load(scan()))) {
DepositFragment frag;
if (resource instanceof Registrar) {
frag = marshaller.marshalRegistrar((Registrar) resource);
} else if (resource instanceof ContactResource) {
frag = marshaller.marshalContact((ContactResource) resource);
} else if (resource instanceof DomainResource) {
DomainResource domain = (DomainResource) resource;
if (!domain.getTld().equals(tld)) {
continue;
}
frag = marshaller.marshalDomain(domain, mode);
} else if (resource instanceof HostResource) {
HostResource host = (HostResource) resource;
frag = host.isSubordinate()
? marshaller.marshalSubordinateHost(
host,
// Note that loadAtPointInTime() does cloneProjectedAtTime(watermark) for us.
loadAtPointInTime(
ofy().load().key(host.getSuperordinateDomain()).now(), watermark).now())
: marshaller.marshalExternalHost(host);
} else {
continue; // Surprise polymorphic entities, e.g. DomainApplication.
}
if (!frag.xml().isEmpty()) {
xmlOutput.write(frag.xml());
counter.increment(frag.type());
}
if (!frag.error().isEmpty()) {
System.err.print(frag.error());
}
}
for (IdnTableEnum idn : IdnTableEnum.values()) {
xmlOutput.write(marshaller.marshalIdn(idn.getTable()));
counter.increment(RdeResourceType.IDN);
}
header = counter.makeHeader(tld, mode);
xmlOutput.write(marshaller.marshalStrictlyOrDie(new XjcRdeHeaderElement(header)));
xmlOutput.write(marshaller.makeFooter());
}
try (OutputStream reportOutputBytes = Files.newOutputStream(reportPath)) {
counter.makeReport(id, watermark, header, revision).marshal(reportOutputBytes, UTF_8);
}
String name = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
encryptor.encrypt(tld, xmlPath, outdir);
Files.move(xmlPath, outdir.resolve(name + ".xml"), REPLACE_EXISTING);
Files.move(reportPath, outdir.resolve(name + "-report.xml"), REPLACE_EXISTING);
} finally {
Files.deleteIfExists(xmlPath);
Files.deleteIfExists(reportPath);
if (tlds.isEmpty()) {
throw new ParameterException("At least one TLD must be specified");
}
}
assertTldsExist(tlds);
private Iterable<EppResource> scan() {
return Iterables.concat(
Iterables.transform(
getEppResourceIndexBuckets(),
new Function<Key<EppResourceIndexBucket>, Iterable<EppResource>>() {
@Override
public Iterable<EppResource> apply(Key<EppResourceIndexBucket> bucket) {
System.err.printf("Scanning EppResourceIndexBucket %d of %d...\n",
bucket.getId(), eppResourceIndexBucketCount);
return scanBucket(bucket);
}}));
}
private Iterable<EppResource> scanBucket(final Key<EppResourceIndexBucket> bucket) {
return ofy().load()
.keys(FluentIterable
.from(mode == RdeMode.FULL
? Arrays.asList(
Key.getKind(ContactResource.class),
Key.getKind(DomainResource.class),
Key.getKind(HostResource.class))
: Arrays.asList(
Key.getKind(DomainResource.class)))
.transformAndConcat(new Function<String, Iterable<EppResourceIndex>>() {
@Override
public Iterable<EppResourceIndex> apply(String kind) {
return ofy().load()
.type(EppResourceIndex.class)
.ancestor(bucket)
.filter("kind", kind)
.iterable();
}})
.transform(new Function<EppResourceIndex, Key<EppResource>>() {
@Override
@SuppressWarnings("unchecked")
public Key<EppResource> apply(EppResourceIndex index) {
return (Key<EppResource>) index.getKey();
}}))
.values();
}
private <T extends EppResource> Iterable<T> load(final Iterable<T> resources) {
return FluentIterable
.from(Iterables.partition(
Iterables.transform(resources,
new Function<T, Result<T>>() {
@Override
public Result<T> apply(T resource) {
return EppResourceUtils.loadAtPointInTime(resource, watermark);
}}),
1000))
.transformAndConcat(new Function<Iterable<Result<T>>, Iterable<T>>() {
@Override
public Iterable<T> apply(Iterable<Result<T>> results) {
return Iterables.transform(results,
new Function<Result<T>, T>() {
@Override
public T apply(Result<T> result) {
return result.now();
}});
}})
.filter(Predicates.notNull());
}
private ImmutableList<Key<EppResourceIndexBucket>> getEppResourceIndexBuckets() {
ImmutableList.Builder<Key<EppResourceIndexBucket>> builder = new ImmutableList.Builder<>();
for (int i = 1; i <= eppResourceIndexBucketCount; i++) {
builder.add(Key.create(EppResourceIndexBucket.class, i));
for (DateTime watermark : watermarks) {
if (!watermark.withTimeAtStartOfDay().equals(watermark)) {
throw new ParameterException("Each watermark date must be the start of a day");
}
}
return builder.build();
if ((revision != null) && (revision < 0)) {
throw new ParameterException("Revision must be greater than or equal to zero");
}
if (outdir.isEmpty()) {
throw new ParameterException("Output subdirectory must not be empty");
}
// Unlike many tool commands, this command is actually invoking an action on the backend module
// (because it's a mapreduce). So we invoke it in a different way.
String hostname = modulesService.getVersionHostname("backend", null);
TaskOptions opts =
withUrl(RdeStagingAction.PATH)
.header("Host", hostname)
.param(RdeModule.PARAM_MANUAL, String.valueOf(true))
.param(RequestParameters.PARAM_TLD, Joiner.on(',').join(tlds))
.param(RdeModule.PARAM_WATERMARK, Joiner.on(',').join(watermarks))
.param(RdeModule.PARAM_MODE, mode.toString())
.param(RdeModule.PARAM_DIRECTORY, outdir);
if (revision != null) {
opts = opts.param(RdeModule.PARAM_REVISION, String.valueOf(revision));
}
queue.add(opts);
}
}

4
java/google/registry/tools/RegistryToolComponent.java
View File

@@ -22,9 +22,11 @@ import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
import google.registry.keyring.api.DummyKeyringModule;
import google.registry.keyring.api.KeyModule;
import google.registry.keyring.kms.KmsModule;
import google.registry.rde.RdeModule;
import google.registry.request.Modules.AppIdentityCredentialModule;
import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.Jackson2Module;
import google.registry.request.Modules.ModulesServiceModule;
import google.registry.request.Modules.URLFetchServiceModule;
import google.registry.request.Modules.UrlFetchTransportModule;
import google.registry.request.Modules.UseAppIdentityCredentialForGoogleApisModule;
@@ -57,6 +59,8 @@ import javax.inject.Singleton;
Jackson2Module.class,
KeyModule.class,
KmsModule.class,
ModulesServiceModule.class,
RdeModule.class,
RegistryToolModule.class,
SystemClockModule.class,
SystemSleeperModule.class,