diff --git a/java/google/registry/module/backend/BackendModule.java b/java/google/registry/module/backend/BackendModule.java index f9a4f24df..57e2ee416 100644 --- a/java/google/registry/module/backend/BackendModule.java +++ b/java/google/registry/module/backend/BackendModule.java @@ -15,10 +15,13 @@ package google.registry.module.backend; import static google.registry.model.registry.Registries.assertTldExists; +import static google.registry.model.registry.Registries.assertTldsExist; import static google.registry.request.RequestParameters.extractOptionalDatetimeParameter; import static google.registry.request.RequestParameters.extractRequiredParameter; +import static google.registry.request.RequestParameters.extractSetOfParameters; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; import dagger.Module; import dagger.Provides; import google.registry.batch.ExpandRecurringBillingEventsAction; @@ -39,6 +42,14 @@ public class BackendModule { return assertTldExists(extractRequiredParameter(req, RequestParameters.PARAM_TLD)); } + @Provides + @Parameter(RequestParameters.PARAM_TLD) + static ImmutableSet provideTlds(HttpServletRequest req) { + ImmutableSet tlds = extractSetOfParameters(req, RequestParameters.PARAM_TLD); + assertTldsExist(tlds); + return tlds; + } + @Provides @Parameter("cursorTime") static Optional provideCursorTime(HttpServletRequest req) { diff --git a/java/google/registry/rde/PendingDeposit.java b/java/google/registry/rde/PendingDeposit.java index bf1dc4855..b0b0f23db 100644 --- a/java/google/registry/rde/PendingDeposit.java +++ b/java/google/registry/rde/PendingDeposit.java @@ -15,6 +15,7 @@ package google.registry.rde; import com.google.auto.value.AutoValue; +import com.google.common.base.Optional; import google.registry.model.common.Cursor.CursorType; import google.registry.model.rde.RdeMode; import java.io.Serializable; @@ -27,15 +28,67 @@ public abstract class PendingDeposit implements Serializable { private static final long serialVersionUID = 3141095605225904433L; + /** + * True if deposits should be generated via manual operation, which does not update the cursor, + * and saves the generated deposits in a special manual subdirectory tree. + */ + public abstract boolean manual(); + + /** TLD for which a deposit should be generated. */ public abstract String tld(); + + /** Watermark date for which a deposit should be generated. */ public abstract DateTime watermark(); + + /** Which type of deposit to generate: full (RDE) or thin (BRDA). */ public abstract RdeMode mode(); - public abstract CursorType cursor(); - public abstract Duration interval(); + + /** The cursor type to update (not used in manual operation). */ + public abstract Optional cursor(); + + /** Amount of time to increment the cursor (not used in manual operation). */ + public abstract Optional interval(); + + /** + * Subdirectory of bucket/manual in which files should be placed, including a trailing slash (used + * only in manual operation). + */ + public abstract Optional directoryWithTrailingSlash(); + + /** + * Revision number for generated files; if absent, use the next available in the sequence (used + * only in manual operation). + */ + public abstract Optional revision(); static PendingDeposit create( String tld, DateTime watermark, RdeMode mode, CursorType cursor, Duration interval) { - return new AutoValue_PendingDeposit(tld, watermark, mode, cursor, interval); + return new AutoValue_PendingDeposit( + false, + tld, + watermark, + mode, + Optional.of(cursor), + Optional.of(interval), + Optional.absent(), + Optional.absent()); + } + + static PendingDeposit createInManualOperation( + String tld, + DateTime watermark, + RdeMode mode, + String directoryWithTrailingSlash, + Optional revision) { + return new AutoValue_PendingDeposit( + true, + tld, + watermark, + mode, + Optional.absent(), + Optional.absent(), + Optional.of(directoryWithTrailingSlash), + revision); } PendingDeposit() {} diff --git a/java/google/registry/rde/RdeModule.java b/java/google/registry/rde/RdeModule.java index 9ff3be9b1..c9dac433d 100644 --- a/java/google/registry/rde/RdeModule.java +++ b/java/google/registry/rde/RdeModule.java @@ -15,12 +15,19 @@ package google.registry.rde; import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; +import static google.registry.request.RequestParameters.extractBooleanParameter; +import static google.registry.request.RequestParameters.extractOptionalIntParameter; +import static google.registry.request.RequestParameters.extractOptionalParameter; +import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter; +import static google.registry.request.RequestParameters.extractSetOfDatetimeParameters; +import static google.registry.request.RequestParameters.extractSetOfParameters; import com.google.appengine.api.taskqueue.Queue; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; import dagger.Module; import dagger.Provides; import google.registry.request.Parameter; -import google.registry.request.RequestParameters; import javax.inject.Named; import javax.servlet.http.HttpServletRequest; import org.joda.time.DateTime; @@ -34,12 +41,45 @@ import org.joda.time.DateTime; public final class RdeModule { static final String PARAM_WATERMARK = "watermark"; - static final String PATH = "path"; + static final String PARAM_MANUAL = "manual"; + static final String PARAM_DIRECTORY = "directory"; + static final String PARAM_MODE = "mode"; + static final String PARAM_REVISION = "revision"; @Provides @Parameter(PARAM_WATERMARK) static DateTime provideWatermark(HttpServletRequest req) { - return DateTime.parse(RequestParameters.extractRequiredParameter(req, PARAM_WATERMARK)); + return extractRequiredDatetimeParameter(req, PARAM_WATERMARK); + } + + @Provides + @Parameter(PARAM_WATERMARK) + static ImmutableSet provideWatermarks(HttpServletRequest req) { + return extractSetOfDatetimeParameters(req, PARAM_WATERMARK); + } + + @Provides + @Parameter(PARAM_MANUAL) + static boolean provideManual(HttpServletRequest req) { + return extractBooleanParameter(req, PARAM_MANUAL); + } + + @Provides + @Parameter(PARAM_DIRECTORY) + static Optional provideDirectory(HttpServletRequest req) { + return extractOptionalParameter(req, PARAM_DIRECTORY); + } + + @Provides + @Parameter(PARAM_MODE) + static ImmutableSet provideMode(HttpServletRequest req) { + return extractSetOfParameters(req, PARAM_MODE); + } + + @Provides + @Parameter(PARAM_REVISION) + static Optional provideRevision(HttpServletRequest req) { + return extractOptionalIntParameter(req, PARAM_REVISION); } @Provides diff --git a/java/google/registry/rde/RdeStagingAction.java b/java/google/registry/rde/RdeStagingAction.java index cf3e4d42e..e2f1e713f 100644 --- a/java/google/registry/rde/RdeStagingAction.java +++ b/java/google/registry/rde/RdeStagingAction.java @@ -17,8 +17,11 @@ package google.registry.rde; import static google.registry.util.PipelineUtils.createJobPath; import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import com.google.common.base.Ascii; +import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Multimaps; import google.registry.config.RegistryConfig.Config; @@ -34,10 +37,14 @@ import google.registry.model.index.EppResourceIndex; import google.registry.model.rde.RdeMode; import google.registry.model.registrar.Registrar; import google.registry.request.Action; +import google.registry.request.HttpException.BadRequestException; +import google.registry.request.Parameter; +import google.registry.request.RequestParameters; import google.registry.request.Response; import google.registry.util.Clock; import google.registry.util.FormattingLogger; import javax.inject.Inject; +import org.joda.time.DateTime; import org.joda.time.Duration; /** @@ -47,7 +54,7 @@ import org.joda.time.Duration; * *

This task starts by asking {@link PendingDepositChecker} which deposits need to be generated. * If there's nothing to deposit, we return 204 No Content; otherwise, we fire off a MapReduce job - * and redirect to its status GUI. + * and redirect to its status GUI. The task can also be run in manual operation, as described below. * *

The mapreduce job scans every {@link EppResource} in Datastore. It maps a point-in-time * representation of each entity to the escrow XML files in which it should appear. @@ -150,6 +157,27 @@ import org.joda.time.Duration; * guarantee referential correctness of your deposits, you must never delete a registrar entity. * * + *

Manual Operation

+ * + *

The task can be run in manual operation by setting certain parameters. Rather than generating + * deposits which are currently outstanding, the task will generate specific deposits. The files + * will be stored in a subdirectory of the "manual" directory, to avoid overwriting regular deposit + * files. Cursors and revision numbers will not be updated, and the upload task will not be kicked + * off. The parameters are: + *

    + *
  • manual: if present and true, manual operation is indicated + *
  • directory: the subdirectory of "manual" into which the files should be placed + *
  • mode: the mode(s) to generate: FULL for RDE deposits, THIN for BRDA deposits + *
  • tld: the tld(s) for which deposits should be generated + *
  • watermark: the date(s) for which deposits should be generated; dates should be start-of-day + *
  • revision: optional; if not specified, the next available revision number will be used + *
+ * + *

The manual, directory, mode, tld and watermark parameters must be present for manual + * operation; they must all be absent for standard operation (except that manual can be present but + * set to false). The revision parameter is optional in manual operation, and must be absent for + * standard operation. + * * @see Registry Data Escrow Specification * @see Domain Name Registration Data Objects Mapping */ @@ -164,23 +192,20 @@ public final class RdeStagingAction implements Runnable { @Inject Response response; @Inject MapreduceRunner mrRunner; @Inject @Config("transactionCooldown") Duration transactionCooldown; + @Inject @Parameter(RdeModule.PARAM_MANUAL) boolean manual; + @Inject @Parameter(RdeModule.PARAM_DIRECTORY) Optional directory; + @Inject @Parameter(RdeModule.PARAM_MODE) ImmutableSet modeStrings; + @Inject @Parameter(RequestParameters.PARAM_TLD) ImmutableSet tlds; + @Inject @Parameter(RdeModule.PARAM_WATERMARK) ImmutableSet watermarks; + @Inject @Parameter(RdeModule.PARAM_REVISION) Optional revision; + @Inject RdeStagingAction() {} @Override public void run() { - ImmutableSetMultimap pendings = ImmutableSetMultimap.copyOf( - Multimaps.filterValues( - pendingDepositChecker.getTldsAndWatermarksPendingDepositForRdeAndBrda(), - new Predicate() { - @Override - public boolean apply(PendingDeposit pending) { - if (clock.nowUtc().isBefore(pending.watermark().plus(transactionCooldown))) { - logger.infofmt("Ignoring within %s cooldown: %s", transactionCooldown, pending); - return false; - } else { - return true; - } - }})); + ImmutableSetMultimap pendings = + manual ? getManualPendingDeposits() : getStandardPendingDeposits(); + if (pendings.isEmpty()) { String message = "Nothing needs to be deposited"; logger.info(message); @@ -188,6 +213,7 @@ public final class RdeStagingAction implements Runnable { response.setPayload(message); return; } + for (PendingDeposit pending : pendings.values()) { logger.infofmt("%s", pending); } @@ -203,4 +229,97 @@ public final class RdeStagingAction implements Runnable { new NullInput(), EppResourceInputs.createEntityInput(EppResource.class))))); } + + private ImmutableSetMultimap getStandardPendingDeposits() { + if (directory.isPresent()) { + throw new BadRequestException("Directory parameter not allowed in standard operation"); + } + if (!modeStrings.isEmpty()) { + throw new BadRequestException("Mode parameter not allowed in standard operation"); + } + if (!tlds.isEmpty()) { + throw new BadRequestException("Tld parameter not allowed in standard operation"); + } + if (!watermarks.isEmpty()) { + throw new BadRequestException("Watermark parameter not allowed in standard operation"); + } + if (revision.isPresent()) { + throw new BadRequestException("Revision parameter not allowed in standard operation"); + } + + return ImmutableSetMultimap.copyOf( + Multimaps.filterValues( + pendingDepositChecker.getTldsAndWatermarksPendingDepositForRdeAndBrda(), + new Predicate() { + @Override + public boolean apply(PendingDeposit pending) { + if (clock.nowUtc().isBefore(pending.watermark().plus(transactionCooldown))) { + logger.infofmt("Ignoring within %s cooldown: %s", transactionCooldown, pending); + return false; + } else { + return true; + } + }})); + } + + private ImmutableSetMultimap getManualPendingDeposits() { + if (!directory.isPresent()) { + throw new BadRequestException("Directory parameter required in manual operation"); + } + if (directory.get().startsWith("/")) { + throw new BadRequestException("Directory must not start with a slash"); + } + String directoryWithTrailingSlash = + directory.get().endsWith("/") ? directory.get() : (directory.get() + '/'); + + if (modeStrings.isEmpty()) { + throw new BadRequestException("Mode parameter required in manual operation"); + } + + ImmutableSet.Builder modesBuilder = new ImmutableSet.Builder<>(); + for (String modeString : modeStrings) { + try { + modesBuilder.add(RdeMode.valueOf(Ascii.toUpperCase(modeString))); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Mode must be FULL for RDE deposits, THIN for BRDA deposits"); + } + } + ImmutableSet modes = modesBuilder.build(); + + if (tlds.isEmpty()) { + throw new BadRequestException("Tld parameter required in manual operation"); + } + + if (watermarks.isEmpty()) { + throw new BadRequestException("Watermark parameter required in manual operation"); + } + // In theory, BRDA deposits should be on a specific day of the week, but in manual mode, let the + // user create deposits on other days. But dates should definitely be at the start of the day; + // otherwise, confusion is likely. + for (DateTime watermark : watermarks) { + if (!watermark.equals(watermark.withTimeAtStartOfDay())) { + throw new BadRequestException("Watermarks must be at the start of a day."); + } + } + + ImmutableSetMultimap.Builder pendingsBuilder = + new ImmutableSetMultimap.Builder<>(); + + for (String tld : tlds) { + for (DateTime watermark : watermarks) { + for (RdeMode mode : modes) { + pendingsBuilder.put( + tld, + PendingDeposit.createInManualOperation( + tld, + watermark, + mode, + directoryWithTrailingSlash, + revision)); + } + } + } + + return pendingsBuilder.build(); + } } diff --git a/java/google/registry/rde/RdeStagingReducer.java b/java/google/registry/rde/RdeStagingReducer.java index 90d8c2b71..91b0d59ac 100644 --- a/java/google/registry/rde/RdeStagingReducer.java +++ b/java/google/registry/rde/RdeStagingReducer.java @@ -17,6 +17,7 @@ package google.registry.rde; import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime; import static google.registry.model.ofy.ObjectifyService.ofy; @@ -107,9 +108,13 @@ public final class RdeStagingReducer extends ReducerDates are parsed as an ISO 8601 timestamp, e.g. {@code + * 1984-12-18TZ}, {@code 2000-01-01T16:20:00Z}. + * + * @throws BadRequestException if one of the parameter values is not a valid {@link DateTime}. + */ + public static ImmutableSet extractSetOfDatetimeParameters( + HttpServletRequest req, String name) { + String[] stringParams = req.getParameterValues(name); + if (stringParams == null) { + return ImmutableSet.of(); + } + ImmutableSet.Builder datesBuilder = new ImmutableSet.Builder<>(); + for (String stringParam : stringParams) { + try { + if (!isNullOrEmpty(stringParam)) { + datesBuilder.add(DateTime.parse(stringParam)); + } + } catch (IllegalArgumentException e) { + throw new BadRequestException("Bad ISO 8601 timestamp: " + name); + } + } + return datesBuilder.build(); + } + /** * Returns first request parameter associated with {@code name} parsed as an optional * {@link InetAddress} (which might be IPv6). diff --git a/java/google/registry/tools/GenerateEscrowDepositCommand.java b/java/google/registry/tools/GenerateEscrowDepositCommand.java index d3eef7d7c..77a2f5567 100644 --- a/java/google/registry/tools/GenerateEscrowDepositCommand.java +++ b/java/google/registry/tools/GenerateEscrowDepositCommand.java @@ -81,7 +81,7 @@ final class GenerateEscrowDepositCommand implements RemoteApiCommand { @Parameter( names = {"-m", "--mode"}, - description = "RDE/BRDA mode of operation.") + description = "FULL/THIN mode of operation.") private RdeMode mode = RdeMode.FULL; @Parameter( diff --git a/javatests/google/registry/rde/RdeStagingActionTest.java b/javatests/google/registry/rde/RdeStagingActionTest.java index f2c1307c2..65a0fd742 100644 --- a/javatests/google/registry/rde/RdeStagingActionTest.java +++ b/javatests/google/registry/rde/RdeStagingActionTest.java @@ -36,6 +36,13 @@ import static java.util.Arrays.asList; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.appengine.tools.cloudstorage.GcsService; import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.appengine.tools.cloudstorage.ListItem; +import com.google.appengine.tools.cloudstorage.ListOptions; +import com.google.appengine.tools.cloudstorage.ListResult; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; @@ -47,7 +54,9 @@ import google.registry.model.common.Cursor.CursorType; import google.registry.model.host.HostResource; import google.registry.model.ofy.Ofy; import google.registry.model.registry.Registry; +import google.registry.request.HttpException.BadRequestException; import google.registry.request.RequestParameters; +import google.registry.testing.ExceptionRule; import google.registry.testing.FakeClock; import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeResponse; @@ -101,6 +110,9 @@ public class RdeStagingActionTest extends MapreduceTestCase { @Rule public final InjectRule inject = new InjectRule(); + @Rule + public final ExceptionRule thrown = new ExceptionRule(); + private final FakeClock clock = new FakeClock(); private final FakeResponse response = new FakeResponse(); private final GcsService gcsService = GcsServiceFactory.createGcsService(); @@ -136,6 +148,47 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.pendingDepositChecker.rdeInterval = Duration.standardDays(1); action.response = response; action.transactionCooldown = Duration.ZERO; + action.directory = Optional.absent(); + action.modeStrings = ImmutableSet.of(); + action.tlds = ImmutableSet.of(); + action.watermarks = ImmutableSet.of(); + action.revision = Optional.absent(); + } + + @Test + public void testRun_modeInNonManualMode_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.modeStrings = ImmutableSet.of("full"); + thrown.expect(BadRequestException.class); + action.run(); + } + + @Test + public void testRun_tldInNonManualMode_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.tlds = ImmutableSet.of("tld"); + thrown.expect(BadRequestException.class); + action.run(); + } + + @Test + public void testRun_watermarkInNonManualMode_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.watermarks = ImmutableSet.of(clock.nowUtc()); + thrown.expect(BadRequestException.class); + action.run(); + } + + @Test + public void testRun_revisionInNonManualMode_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.revision = Optional.of(42); + thrown.expect(BadRequestException.class); + action.run(); } @Test @@ -184,6 +237,86 @@ public class RdeStagingActionTest extends MapreduceTestCase { assertAtLeastOneTaskIsEnqueued("mapreduce"); } + @Test + public void testManualRun_emptyMode_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.manual = true; + action.directory = Optional.of("test/"); + action.modeStrings = ImmutableSet.of(); + action.tlds = ImmutableSet.of("lol"); + action.watermarks = ImmutableSet.of(clock.nowUtc()); + thrown.expect(BadRequestException.class); + action.run(); + } + + @Test + public void testManualRun_invalidMode_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.manual = true; + action.directory = Optional.of("test/"); + action.modeStrings = ImmutableSet.of("full", "thing"); + action.tlds = ImmutableSet.of("lol"); + action.watermarks = ImmutableSet.of(clock.nowUtc()); + thrown.expect(BadRequestException.class); + action.run(); + } + + @Test + public void testManualRun_emptyTld_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.manual = true; + action.directory = Optional.of("test/"); + action.modeStrings = ImmutableSet.of("full"); + action.tlds = ImmutableSet.of(); + action.watermarks = ImmutableSet.of(clock.nowUtc()); + thrown.expect(BadRequestException.class); + action.run(); + } + + @Test + public void testManualRun_emptyWatermark_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.manual = true; + action.directory = Optional.of("test/"); + action.modeStrings = ImmutableSet.of("full"); + action.tlds = ImmutableSet.of("lol"); + action.watermarks = ImmutableSet.of(); + thrown.expect(BadRequestException.class); + action.run(); + } + + @Test + public void testManualRun_nonDayStartWatermark_throwsException() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.manual = true; + action.directory = Optional.of("test/"); + action.modeStrings = ImmutableSet.of("full"); + action.tlds = ImmutableSet.of("lol"); + action.watermarks = ImmutableSet.of(DateTime.parse("2001-01-01T01:36:45Z")); + thrown.expect(BadRequestException.class); + action.run(); + } + + @Test + public void testManualRun_validParameters_runsMapReduce() throws Exception { + createTldWithEscrowEnabled("lol"); + clock.setTo(DateTime.parse("2000-01-01TZ")); + action.manual = true; + action.directory = Optional.of("test/"); + action.modeStrings = ImmutableSet.of("full"); + action.tlds = ImmutableSet.of("lol"); + action.watermarks = ImmutableSet.of(DateTime.parse("2001-01-01TZ")); + action.run(); + assertThat(response.getStatus()).isEqualTo(200); + assertThat(response.getPayload()).contains("_ah/pipeline/status.html?root="); + assertAtLeastOneTaskIsEnqueued("mapreduce"); + } + @Test public void testMapReduce_bunchOfResources_headerHasCorrectCounts() throws Exception { clock.setTo(DateTime.parse("1999-12-31TZ")); @@ -610,6 +743,87 @@ public class RdeStagingActionTest extends MapreduceTestCase { .isEqualTo(DateTime.parse("1984-12-21TZ")); } + private void doManualModeMapReduceTest(int revision, ImmutableSet tlds) throws Exception { + clock.setTo(DateTime.parse("1999-12-31TZ")); + for (String tld : tlds) { + createTldWithEscrowEnabled(tld); + makeDomainResource(clock, tld); + setCursor(Registry.get(tld), RDE_STAGING, DateTime.parse("1999-01-01TZ")); + setCursor(Registry.get(tld), BRDA, DateTime.parse("2001-01-01TZ")); + } + + action.manual = true; + action.directory = Optional.of("test/"); + action.modeStrings = ImmutableSet.of("full", "thin"); + action.tlds = tlds; + action.watermarks = + ImmutableSet.of(DateTime.parse("2000-01-01TZ"), DateTime.parse("2000-01-02TZ")); + action.revision = Optional.of(revision); + + action.run(); + executeTasksUntilEmpty("mapreduce", clock); + + ListResult listResult = + gcsService.list("rde-bucket", new ListOptions.Builder().setPrefix("manual/test").build()); + ImmutableSet filenames = + FluentIterable.from(ImmutableList.copyOf(listResult)) + .transform( + new Function() { + @Override + public String apply(ListItem listItem) { + return listItem.getName(); + } + }) + .toSet(); + for (String tld : tlds) { + assertThat(filenames) + .containsAllOf( + "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde", + "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde", + "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.length", + "manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde", + "manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length", + "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde", + "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde", + "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.length", + "manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde", + "manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length"); + + assertThat( + ofy() + .load() + .key(Cursor.createKey(RDE_STAGING, Registry.get(tld))) + .now() + .getCursorTime()) + .isEqualTo(DateTime.parse("1999-01-01TZ")); + assertThat(ofy().load().key(Cursor.createKey(BRDA, Registry.get(tld))).now().getCursorTime()) + .isEqualTo(DateTime.parse("2001-01-01TZ")); + } + } + + @Test + public void testMapReduce_manualMode_generatesCorrectDepositsWithoutAdvancingCursors() + throws Exception { + doManualModeMapReduceTest(0, ImmutableSet.of("lol")); + XmlTestUtils.assertXmlEquals( + readResourceUtf8(getClass(), "testdata/testMapReduce_withDomain_producesExpectedXml.xml"), + readXml("manual/test/lol_2000-01-01_full_S1_R0.xml.ghostryde"), + "deposit.contents.registrar.crDate", + "deposit.contents.registrar.upDate"); + XmlTestUtils.assertXmlEquals( + readResourceUtf8(getClass(), "testdata/testMapReduce_withDomain_producesReportXml.xml"), + readXml( + "manual/test/lol_2000-01-01_full_S1_R0-report.xml.ghostryde"), + "deposit.contents.registrar.crDate", + "deposit.contents.registrar.upDate"); + } + + @Test + public void testMapReduce_manualMode_nonZeroRevisionAndMultipleTlds() + throws Exception { + doManualModeMapReduceTest(42, ImmutableSet.of("lol", "slug")); + } + private String readXml(String objectName) throws IOException, PGPException { GcsFilename file = new GcsFilename("rde-bucket", objectName); return new String(Ghostryde.decode(readGcsFile(gcsService, file), decryptKey).getData(), UTF_8);