diff --git a/java/google/registry/beam/BillingEvent.java b/java/google/registry/beam/BillingEvent.java index 30879b2fa..4f728d8b8 100644 --- a/java/google/registry/beam/BillingEvent.java +++ b/java/google/registry/beam/BillingEvent.java @@ -171,8 +171,8 @@ public abstract class BillingEvent implements Serializable { *

When modifying this function, take care to ensure that there's no way to generate an illegal * filepath with the arguments, such as "../sensitive_info". */ - String toFilename() { - return String.format("%s_%s", registrarId(), tld()); + String toFilename(String yearMonth) { + return String.format("invoice_details_%s_%s_%s", yearMonth, registrarId(), tld()); } /** Generates a CSV representation of this {@code BillingEvent}. */ diff --git a/java/google/registry/beam/InvoicingPipeline.java b/java/google/registry/beam/InvoicingPipeline.java index f47c0a40c..3ec043cdf 100644 --- a/java/google/registry/beam/InvoicingPipeline.java +++ b/java/google/registry/beam/InvoicingPipeline.java @@ -17,6 +17,7 @@ package google.registry.beam; import google.registry.beam.BillingEvent.InvoiceGroupingKey; import google.registry.beam.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder; import google.registry.config.RegistryConfig.Config; +import java.io.Serializable; import javax.inject.Inject; import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; @@ -29,6 +30,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.options.Description; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.ValueProvider; +import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider; import org.apache.beam.sdk.transforms.Count; import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.PTransform; @@ -49,7 +51,7 @@ import org.apache.beam.sdk.values.TypeDescriptors; * * @see Dataflow Templates */ -public class InvoicingPipeline { +public class InvoicingPipeline implements Serializable { @Inject @Config("projectId") String projectId; @Inject @Config("apacheBeamBucketUrl") String beamBucket; @@ -66,12 +68,12 @@ public class InvoicingPipeline { /** Deploys the invoicing pipeline as a template on GCS, for a given projectID and GCS bucket. */ public void deploy() { + // We can't store options as a member variable due to serialization concerns. InvoicingPipelineOptions options = PipelineOptionsFactory.as(InvoicingPipelineOptions.class); options.setProject(projectId); options.setRunner(DataflowRunner.class); options.setStagingLocation(beamBucket + "/staging"); options.setTemplateLocation(beamBucket + "/templates/invoicing"); - Pipeline p = Pipeline.create(options); PCollection billingEvents = @@ -81,9 +83,9 @@ public class InvoicingPipeline { .fromQuery(InvoicingUtils.makeQueryProvider(options.getYearMonth(), projectId)) .withCoder(SerializableCoder.of(BillingEvent.class)) .usingStandardSql() - .withoutValidation()); - - applyTerminalTransforms(billingEvents); + .withoutValidation() + .withTemplateCompatibility()); + applyTerminalTransforms(billingEvents, options.getYearMonth()); p.run(); } @@ -92,13 +94,15 @@ public class InvoicingPipeline { * *

This is factored out purely to facilitate testing. */ - void applyTerminalTransforms(PCollection billingEvents) { + void applyTerminalTransforms( + PCollection billingEvents, ValueProvider yearMonthProvider) { billingEvents.apply( - "Write events to separate CSVs keyed by registrarId_tld pair", writeDetailReports()); + "Write events to separate CSVs keyed by registrarId_tld pair", + writeDetailReports(yearMonthProvider)); billingEvents .apply("Generate overall invoice rows", new GenerateInvoiceRows()) - .apply("Write overall invoice to CSV", writeInvoice()); + .apply("Write overall invoice to CSV", writeInvoice(yearMonthProvider)); } /** Transform that converts a {@code BillingEvent} into an invoice CSV row. */ @@ -121,19 +125,23 @@ public class InvoicingPipeline { } /** Returns an IO transform that writes the overall invoice to a single CSV file. */ - private TextIO.Write writeInvoice() { + private TextIO.Write writeInvoice(ValueProvider yearMonthProvider) { return TextIO.write() - .to(beamBucket + "/results/overall_invoice") + .to( + NestedValueProvider.of( + yearMonthProvider, + yearMonth -> String.format("%s/results/CRR-INV-%s", beamBucket, yearMonth))) .withHeader(InvoiceGroupingKey.invoiceHeader()) .withoutSharding() .withSuffix(".csv"); } /** Returns an IO transform that writes detail reports to registrar-tld keyed CSV files. */ - private TextIO.TypedWrite writeDetailReports() { + private TextIO.TypedWrite writeDetailReports( + ValueProvider yearMonthProvider) { return TextIO.writeCustomType() .to( - InvoicingUtils.makeDestinationFunction(beamBucket + "/results"), + InvoicingUtils.makeDestinationFunction(beamBucket + "/results", yearMonthProvider), InvoicingUtils.makeEmptyDestinationParams(beamBucket + "/results")) .withFormatFunction(BillingEvent::toCsv) .withoutSharding() diff --git a/java/google/registry/beam/InvoicingUtils.java b/java/google/registry/beam/InvoicingUtils.java index 2293d109e..ea2abbb59 100644 --- a/java/google/registry/beam/InvoicingUtils.java +++ b/java/google/registry/beam/InvoicingUtils.java @@ -39,15 +39,23 @@ public class InvoicingUtils { * Returns a function mapping from {@code BillingEvent} to filename {@code Params}. * *

Beam uses this to determine which file a given {@code BillingEvent} should get placed into. + * + * @param outputBucket the GCS bucket we're outputting reports to + * @param yearMonthProvider a runtime provider for the yyyy-MM we're generating the invoice for */ - static SerializableFunction makeDestinationFunction(String outputBucket) { + static SerializableFunction makeDestinationFunction( + String outputBucket, ValueProvider yearMonthProvider) { return billingEvent -> new Params() .withShardTemplate("") .withSuffix(".csv") .withBaseFilename( - FileBasedSink.convertToFileResourceIfPossible( - String.format("%s/%s", outputBucket, billingEvent.toFilename()))); + NestedValueProvider.of( + yearMonthProvider, + yearMonth -> + FileBasedSink.convertToFileResourceIfPossible( + String.format( + "%s/%s", outputBucket, billingEvent.toFilename(yearMonth))))); } /** diff --git a/java/google/registry/billing/BUILD b/java/google/registry/billing/BUILD index a9bc84009..a601a7052 100644 --- a/java/google/registry/billing/BUILD +++ b/java/google/registry/billing/BUILD @@ -17,6 +17,7 @@ java_library( "//java/google/registry/util", "@com_google_api_client_appengine", "@com_google_apis_google_api_services_dataflow", + "@com_google_appengine_api_1_0_sdk", "@com_google_dagger", "@com_google_guava", "@com_google_http_client", diff --git a/java/google/registry/billing/BillingModule.java b/java/google/registry/billing/BillingModule.java index d0d99ae0d..4f585d181 100644 --- a/java/google/registry/billing/BillingModule.java +++ b/java/google/registry/billing/BillingModule.java @@ -14,6 +14,8 @@ package google.registry.billing; +import static google.registry.request.RequestParameters.extractRequiredParameter; + import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; @@ -22,8 +24,10 @@ import com.google.common.collect.ImmutableSet; import dagger.Module; import dagger.Provides; import google.registry.config.RegistryConfig.Config; +import google.registry.request.Parameter; import java.util.Set; import java.util.function.Function; +import javax.servlet.http.HttpServletRequest; /** Module for dependencies required by monthly billing actions. */ @Module @@ -31,6 +35,15 @@ public final class BillingModule { private static final String CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"; + static final String BILLING_QUEUE = "billing"; + static final String PARAM_JOB_ID = "jobId"; + + /** Provides the invoicing Dataflow jobId enqueued by {@link GenerateInvoicesAction}. */ + @Provides + @Parameter(PARAM_JOB_ID) + static String provideJobId(HttpServletRequest req) { + return extractRequiredParameter(req, PARAM_JOB_ID); + } /** Constructs a {@link Dataflow} API client with default settings. */ @Provides diff --git a/java/google/registry/billing/GenerateInvoicesAction.java b/java/google/registry/billing/GenerateInvoicesAction.java index 8e5822762..8c75b54ed 100644 --- a/java/google/registry/billing/GenerateInvoicesAction.java +++ b/java/google/registry/billing/GenerateInvoicesAction.java @@ -22,6 +22,9 @@ import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.LaunchTemplateParameters; import com.google.api.services.dataflow.model.LaunchTemplateResponse; import com.google.api.services.dataflow.model.RuntimeEnvironment; +import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.TaskOptions; +import com.google.common.collect.ImmutableMap; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.request.Action; @@ -30,9 +33,12 @@ import google.registry.request.auth.Auth; import google.registry.util.FormattingLogger; import java.io.IOException; import javax.inject.Inject; +import org.joda.time.Duration; +import org.joda.time.YearMonth; /** - * Invokes the {@code InvoicingPipeline} beam template via the REST api. + * Invokes the {@code InvoicingPipeline} beam template via the REST api, and enqueues the {@link + * PublishInvoicesAction} to publish the subsequent output. * *

This action runs the {@link google.registry.beam.InvoicingPipeline} beam template, staged at * gs://-beam/templates/invoicing. The pipeline then generates invoices for the month and @@ -43,25 +49,35 @@ public class GenerateInvoicesAction implements Runnable { private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); - @Inject @Config("projectId") String projectId; - @Inject @Config("apacheBeamBucketUrl") String beamBucketUrl; + @Inject + @Config("projectId") + String projectId; + + @Inject + @Config("apacheBeamBucketUrl") + String beamBucketUrl; + + @Inject YearMonth yearMonth; @Inject Dataflow dataflow; @Inject Response response; - @Inject GenerateInvoicesAction() {} + + @Inject + GenerateInvoicesAction() {} static final String PATH = "/_dr/task/generateInvoices"; @Override public void run() { - logger.info("Launching dataflow job"); + logger.infofmt("Launching invoicing pipeline for %s", yearMonth); try { LaunchTemplateParameters params = new LaunchTemplateParameters() - .setJobName("test-invoicing") + .setJobName(String.format("invoicing-%s", yearMonth)) .setEnvironment( new RuntimeEnvironment() .setZone("us-east1-c") - .setTempLocation(beamBucketUrl + "/temp")); + .setTempLocation(beamBucketUrl + "/temporary")) + .setParameters(ImmutableMap.of("yearMonth", yearMonth.toString("yyyy-MM"))); LaunchTemplateResponse launchResponse = dataflow .projects() @@ -70,6 +86,14 @@ public class GenerateInvoicesAction implements Runnable { .setGcsPath(beamBucketUrl + "/templates/invoicing") .execute(); logger.infofmt("Got response: %s", launchResponse.getJob().toPrettyString()); + String jobId = launchResponse.getJob().getId(); + TaskOptions uploadTask = + TaskOptions.Builder.withUrl(PublishInvoicesAction.PATH) + .method(TaskOptions.Method.POST) + // Dataflow jobs tend to take about 10 minutes to complete. + .countdownMillis(Duration.standardMinutes(10).getMillis()) + .param(BillingModule.PARAM_JOB_ID, jobId); + QueueFactory.getQueue(BillingModule.BILLING_QUEUE).add(uploadTask); } catch (IOException e) { logger.warningfmt("Template Launch failed due to: %s", e.getMessage()); response.setStatus(SC_INTERNAL_SERVER_ERROR); diff --git a/java/google/registry/billing/PublishInvoicesAction.java b/java/google/registry/billing/PublishInvoicesAction.java new file mode 100644 index 000000000..0e4b61f06 --- /dev/null +++ b/java/google/registry/billing/PublishInvoicesAction.java @@ -0,0 +1,89 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.billing; + +import static google.registry.request.Action.Method.POST; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; +import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.Job; +import com.google.common.net.MediaType; +import google.registry.config.RegistryConfig.Config; +import google.registry.request.Action; +import google.registry.request.Parameter; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import google.registry.util.FormattingLogger; +import java.io.IOException; +import javax.inject.Inject; + +/** + * Uploads the results of the {@link google.registry.beam.InvoicingPipeline}. + * + *

This relies on the retry semantics in {@code queue.xml} to ensure proper upload, in spite of + * fluctuations in generation timing. + * + * @see + * Job States + */ +@Action(path = PublishInvoicesAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN) +public class PublishInvoicesAction implements Runnable { + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + private static final String JOB_DONE = "JOB_STATE_DONE"; + private static final String JOB_FAILED = "JOB_STATE_FAILED"; + + @Inject @Config("projectId") String projectId; + @Inject @Parameter(BillingModule.PARAM_JOB_ID) String jobId; + @Inject Dataflow dataflow; + @Inject Response response; + @Inject PublishInvoicesAction() {} + + static final String PATH = "/_dr/task/publishInvoices"; + + @Override + public void run() { + logger.info("Starting publish job."); + try { + Job job = dataflow.projects().jobs().get(projectId, jobId).execute(); + String state = job.getCurrentState(); + switch (state) { + case JOB_DONE: + logger.infofmt("Dataflow job %s finished successfully.", jobId); + response.setStatus(SC_OK); + // TODO(larryruili): Implement upload logic. + break; + case JOB_FAILED: + logger.severefmt("Dataflow job %s finished unsuccessfully.", jobId); + // Return a 'success' code to stop task queue retry. + response.setStatus(SC_NO_CONTENT); + // TODO(larryruili): Implement failure response. + break; + default: + logger.infofmt("Job in non-terminal state %s, retrying:", state); + response.setStatus(SC_NOT_MODIFIED); + break; + } + } catch (IOException e) { + logger.warningfmt("Template Launch failed due to: %s", e.getMessage()); + response.setStatus(SC_INTERNAL_SERVER_ERROR); + response.setContentType(MediaType.PLAIN_TEXT_UTF_8); + response.setPayload(String.format("Template launch failed: %s", e.getMessage())); + } + } +} diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index d37a3780a..44d879c4b 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -68,6 +68,15 @@ /_dr/task/generateInvoices + + + backend-servlet + /_dr/task/publishInvoices + + + + billing + 1/m + 1 + + 5 + 180 + 180 + + + diff --git a/java/google/registry/module/backend/BackendModule.java b/java/google/registry/module/backend/BackendModule.java index c617c7d86..d4d564871 100644 --- a/java/google/registry/module/backend/BackendModule.java +++ b/java/google/registry/module/backend/BackendModule.java @@ -17,6 +17,7 @@ package google.registry.module.backend; import static google.registry.model.registry.Registries.assertTldExists; import static google.registry.model.registry.Registries.assertTldsExist; import static google.registry.request.RequestParameters.extractOptionalDatetimeParameter; +import static google.registry.request.RequestParameters.extractOptionalParameter; import static google.registry.request.RequestParameters.extractRequiredParameter; import static google.registry.request.RequestParameters.extractSetOfParameters; @@ -24,11 +25,16 @@ import com.google.common.collect.ImmutableSet; import dagger.Module; import dagger.Provides; import google.registry.batch.ExpandRecurringBillingEventsAction; +import google.registry.request.HttpException.BadRequestException; import google.registry.request.Parameter; import google.registry.request.RequestParameters; +import google.registry.util.Clock; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import org.joda.time.DateTime; +import org.joda.time.YearMonth; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; /** * Dagger module for injecting common settings for all Backend tasks. @@ -36,6 +42,8 @@ import org.joda.time.DateTime; @Module public class BackendModule { + public static final String PARAM_YEAR_MONTH = "yearMonth"; + @Provides @Parameter(RequestParameters.PARAM_TLD) static String provideTld(HttpServletRequest req) { @@ -56,4 +64,30 @@ public class BackendModule { return extractOptionalDatetimeParameter( req, ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME); } + + /** Extracts an optional YearMonth in yyyy-MM format from the request. */ + @Provides + @Parameter(PARAM_YEAR_MONTH) + static Optional provideYearMonthOptional(HttpServletRequest req) { + DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM"); + Optional optionalYearMonthStr = extractOptionalParameter(req, PARAM_YEAR_MONTH); + try { + return optionalYearMonthStr.map(s -> YearMonth.parse(s, formatter)); + } catch (IllegalArgumentException e) { + throw new BadRequestException( + String.format( + "yearMonth must be in yyyy-MM format, got %s instead", + optionalYearMonthStr.orElse("UNSPECIFIED YEARMONTH"))); + } + } + + /** + * Provides the yearMonth in yyyy-MM format, if not specified in the request, defaults to one + * month prior to run time. + */ + @Provides + static YearMonth provideYearMonth( + @Parameter(PARAM_YEAR_MONTH) Optional yearMonthOptional, Clock clock) { + return yearMonthOptional.orElseGet(() -> new YearMonth(clock.nowUtc().minusMonths(1))); + } } diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index e2f291873..03d32e643 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -30,6 +30,7 @@ import google.registry.batch.ResaveAllEppResourcesAction; import google.registry.batch.VerifyEntityIntegrityAction; import google.registry.billing.BillingModule; import google.registry.billing.GenerateInvoicesAction; +import google.registry.billing.PublishInvoicesAction; import google.registry.cron.CommitLogFanoutAction; import google.registry.cron.CronModule; import google.registry.cron.TldFanoutAction; @@ -147,6 +148,7 @@ interface BackendRequestComponent { TmchDnlAction tmchDnlAction(); TmchSmdrlAction tmchSmdrlAction(); UpdateSnapshotViewAction updateSnapshotViewAction(); + PublishInvoicesAction uploadInvoicesAction(); VerifyEntityIntegrityAction verifyEntityIntegrityAction(); @Subcomponent.Builder diff --git a/java/google/registry/reporting/IcannReportingModule.java b/java/google/registry/reporting/IcannReportingModule.java index 726b5515b..73608fa00 100644 --- a/java/google/registry/reporting/IcannReportingModule.java +++ b/java/google/registry/reporting/IcannReportingModule.java @@ -28,7 +28,6 @@ import dagger.Provides; import google.registry.bigquery.BigqueryConnection; import google.registry.request.HttpException.BadRequestException; import google.registry.request.Parameter; -import google.registry.util.Clock; import google.registry.util.SendEmailService; import java.lang.annotation.Documented; import java.lang.annotation.Retention; @@ -38,7 +37,6 @@ import javax.servlet.http.HttpServletRequest; import org.joda.time.Duration; import org.joda.time.YearMonth; import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; /** Module for dependencies required by ICANN monthly transactions/activity reporting. */ @Module @@ -50,7 +48,6 @@ public final class IcannReportingModule { ACTIVITY } - static final String PARAM_YEAR_MONTH = "yearMonth"; static final String PARAM_SUBDIR = "subdir"; static final String PARAM_REPORT_TYPE = "reportType"; static final String ICANN_REPORTING_DATA_SET = "icann_reporting"; @@ -59,29 +56,6 @@ public final class IcannReportingModule { private static final String DEFAULT_SUBDIR = "icann/monthly"; private static final String BIGQUERY_SCOPE = "https://www.googleapis.com/auth/cloud-platform"; - /** Extracts an optional YearMonth in yyyy-MM format from the request. */ - @Provides - @Parameter(PARAM_YEAR_MONTH) - static Optional provideYearMonthOptional(HttpServletRequest req) { - DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM"); - Optional optionalYearMonthStr = extractOptionalParameter(req, PARAM_YEAR_MONTH); - try { - return optionalYearMonthStr.map(s -> YearMonth.parse(s, formatter)); - } catch (IllegalArgumentException e) { - throw new BadRequestException( - String.format( - "yearMonth must be in yyyy-MM format, got %s instead", - optionalYearMonthStr.orElse("UNSPECIFIED YEARMONTH"))); - } - } - - /** Provides the yearMonth in yyyy-MM format, defaults to one month prior to run time. */ - @Provides - static YearMonth provideYearMonth( - @Parameter(PARAM_YEAR_MONTH) Optional yearMonthOptional, Clock clock) { - return yearMonthOptional.orElseGet(() -> new YearMonth(clock.nowUtc().minusMonths(1))); - } - /** Provides an optional subdirectory to store/upload reports to, extracted from the request. */ @Provides @Parameter(PARAM_SUBDIR) diff --git a/java/google/registry/reporting/IcannReportingStagingAction.java b/java/google/registry/reporting/IcannReportingStagingAction.java index 921f47dff..07ab5363d 100644 --- a/java/google/registry/reporting/IcannReportingStagingAction.java +++ b/java/google/registry/reporting/IcannReportingStagingAction.java @@ -35,7 +35,6 @@ import google.registry.util.Retrier; import javax.inject.Inject; import org.joda.time.Duration; import org.joda.time.YearMonth; -import org.joda.time.format.DateTimeFormat; /** * Action that generates monthly ICANN activity and transactions reports. @@ -98,9 +97,6 @@ public final class IcannReportingStagingAction implements Runnable { TaskOptions uploadTask = TaskOptions.Builder.withUrl(IcannReportingUploadAction.PATH) .method(Method.POST) .countdownMillis(Duration.standardMinutes(2).getMillis()) - .param( - IcannReportingModule.PARAM_YEAR_MONTH, - DateTimeFormat.forPattern("yyyy-MM").print(yearMonth)) .param(IcannReportingModule.PARAM_SUBDIR, subdir); QueueFactory.getQueue(CRON_QUEUE).add(uploadTask); return null; diff --git a/javatests/google/registry/beam/BillingEventTest.java b/javatests/google/registry/beam/BillingEventTest.java index 2b210ef8f..bbe94da52 100644 --- a/javatests/google/registry/beam/BillingEventTest.java +++ b/javatests/google/registry/beam/BillingEventTest.java @@ -116,7 +116,7 @@ public class BillingEventTest { @Test public void testGenerateBillingEventFilename() { BillingEvent event = BillingEvent.parseFromRecord(schemaAndRecord); - assertThat(event.toFilename()).isEqualTo("myRegistrar_test"); + assertThat(event.toFilename("2017-10")).isEqualTo("invoice_details_2017-10_myRegistrar_test"); } @Test diff --git a/javatests/google/registry/beam/InvoicingPipelineTest.java b/javatests/google/registry/beam/InvoicingPipelineTest.java index 944172e06..cb6b47a28 100644 --- a/javatests/google/registry/beam/InvoicingPipelineTest.java +++ b/javatests/google/registry/beam/InvoicingPipelineTest.java @@ -26,6 +26,7 @@ import java.util.Map.Entry; import org.apache.beam.runners.direct.DirectRunner; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; @@ -124,17 +125,17 @@ public class InvoicingPipelineTest { /** Returns a map from filename to expected contents for detail reports. */ private ImmutableMap> getExpectedDetailReportMap() { return ImmutableMap.of( - "theRegistrar_test.csv", + "invoice_details_2017-10_theRegistrar_test.csv", ImmutableList.of( "1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234," + "test,RENEW,mydomain2.test,REPO-ID,3,USD,20.50,", "1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234," + "test,RENEW,mydomain.test,REPO-ID,3,USD,20.50,"), - "theRegistrar_hello.csv", + "invoice_details_2017-10_theRegistrar_hello.csv", ImmutableList.of( "1,2017-10-02 00:00:00 UTC,2017-09-29 00:00:00 UTC,theRegistrar,234," + "hello,CREATE,mydomain3.hello,REPO-ID,5,JPY,70.75,"), - "googledomains_test.csv", + "invoice_details_2017-10_googledomains_test.csv", ImmutableList.of( "1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,googledomains,456," + "test,RENEW,mydomain4.test,REPO-ID,1,USD,20.50,")); @@ -154,7 +155,7 @@ public class InvoicingPipelineTest { public void testEndToEndPipeline_generatesExpectedFiles() throws Exception { ImmutableList inputRows = getInputEvents(); PCollection input = p.apply(Create.of(inputRows)); - invoicingPipeline.applyTerminalTransforms(input); + invoicingPipeline.applyTerminalTransforms(input, StaticValueProvider.of("2017-10")); p.run(); for (Entry> entry : getExpectedDetailReportMap().entrySet()) { @@ -166,7 +167,7 @@ public class InvoicingPipelineTest { .containsExactlyElementsIn(entry.getValue()); } - ImmutableList overallInvoice = resultFileContents("overall_invoice.csv"); + ImmutableList overallInvoice = resultFileContents("CRR-INV-2017-10.csv"); assertThat(overallInvoice.get(0)) .isEqualTo( "StartDate,EndDate,ProductAccountKey,Amount,AmountCurrency,BillingProductCode," @@ -179,7 +180,7 @@ public class InvoicingPipelineTest { /** Returns the text contents of a file under the beamBucket/results directory. */ private ImmutableList resultFileContents(String filename) throws Exception { File resultFile = - new File(String.format("%s/results/%s", invoicingPipeline.beamBucket, filename)); + new File(String.format("%s/results/%s", tempFolder.getRoot().getAbsolutePath(), filename)); return ImmutableList.copyOf( ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n")); } diff --git a/javatests/google/registry/beam/InvoicingUtilsTest.java b/javatests/google/registry/beam/InvoicingUtilsTest.java index 7100c36a8..01e3f6f86 100644 --- a/javatests/google/registry/beam/InvoicingUtilsTest.java +++ b/javatests/google/registry/beam/InvoicingUtilsTest.java @@ -15,6 +15,7 @@ package google.registry.beam; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -35,11 +36,11 @@ public class InvoicingUtilsTest { @Test public void testDestinationFunction_generatesProperFileParams() { SerializableFunction destinationFunction = - InvoicingUtils.makeDestinationFunction("my/directory"); + InvoicingUtils.makeDestinationFunction("my/directory", StaticValueProvider.of("2017-10")); BillingEvent billingEvent = mock(BillingEvent.class); // We mock BillingEvent to make the test independent of the implementation of toFilename() - when(billingEvent.toFilename()).thenReturn("registrar_tld"); + when(billingEvent.toFilename(any())).thenReturn("invoice_details_2017-10_registrar_tld"); assertThat(destinationFunction.apply(billingEvent)) .isEqualTo( @@ -47,7 +48,8 @@ public class InvoicingUtilsTest { .withShardTemplate("") .withSuffix(".csv") .withBaseFilename( - FileBasedSink.convertToFileResourceIfPossible("my/directory/registrar_tld"))); + FileBasedSink.convertToFileResourceIfPossible( + "my/directory/invoice_details_2017-10_registrar_tld"))); } @Test diff --git a/javatests/google/registry/billing/BUILD b/javatests/google/registry/billing/BUILD index 7ffa8b0e7..a1e08787d 100644 --- a/javatests/google/registry/billing/BUILD +++ b/javatests/google/registry/billing/BUILD @@ -18,6 +18,7 @@ java_library( "@com_google_guava", "@com_google_truth", "@com_google_truth_extensions_truth_java8_extension", + "@javax_servlet_api", "@joda_time", "@junit", "@org_apache_beam_runners_direct_java", diff --git a/javatests/google/registry/billing/GenerateInvoicesActionTest.java b/javatests/google/registry/billing/GenerateInvoicesActionTest.java index 23716a7db..11a0d4e7a 100644 --- a/javatests/google/registry/billing/GenerateInvoicesActionTest.java +++ b/javatests/google/registry/billing/GenerateInvoicesActionTest.java @@ -15,6 +15,7 @@ package google.registry.billing; import static com.google.common.truth.Truth.assertThat; +import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -28,9 +29,14 @@ import com.google.api.services.dataflow.model.Job; import com.google.api.services.dataflow.model.LaunchTemplateParameters; import com.google.api.services.dataflow.model.LaunchTemplateResponse; import com.google.api.services.dataflow.model.RuntimeEnvironment; +import com.google.common.collect.ImmutableMap; +import google.registry.testing.AppEngineRule; import google.registry.testing.FakeResponse; +import google.registry.testing.TaskQueueHelper.TaskMatcher; import java.io.IOException; +import org.joda.time.YearMonth; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -46,6 +52,9 @@ public class GenerateInvoicesActionTest { GenerateInvoicesAction action; FakeResponse response = new FakeResponse(); + @Rule + public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build(); + @Before public void initializeObjects() throws Exception { when(dataflow.projects()).thenReturn(projects); @@ -53,13 +62,16 @@ public class GenerateInvoicesActionTest { when(templates.launch(any(String.class), any(LaunchTemplateParameters.class))) .thenReturn(launch); when(launch.setGcsPath(any(String.class))).thenReturn(launch); - when(launch.execute()).thenReturn(new LaunchTemplateResponse().setJob(new Job())); + Job job = new Job(); + job.setId("12345"); + when(launch.execute()).thenReturn(new LaunchTemplateResponse().setJob(job)); action = new GenerateInvoicesAction(); action.dataflow = dataflow; action.response = response; action.projectId = "test-project"; action.beamBucketUrl = "gs://test-project-beam"; + action.yearMonth = new YearMonth(2017, 10); } @Test @@ -67,15 +79,23 @@ public class GenerateInvoicesActionTest { action.run(); LaunchTemplateParameters expectedParams = new LaunchTemplateParameters() - .setJobName("test-invoicing") + .setJobName("invoicing-2017-10") .setEnvironment( new RuntimeEnvironment() .setZone("us-east1-c") - .setTempLocation("gs://test-project-beam/temp")); + .setTempLocation("gs://test-project-beam/temporary")) + .setParameters(ImmutableMap.of("yearMonth", "2017-10")); verify(templates).launch("test-project", expectedParams); verify(launch).setGcsPath("gs://test-project-beam/templates/invoicing"); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getPayload()).isEqualTo("Launched dataflow template."); + + TaskMatcher matcher = + new TaskMatcher() + .url("/_dr/task/publishInvoices") + .method("POST") + .param("jobId", "12345"); + assertTasksEnqueued("billing", matcher); } @Test diff --git a/javatests/google/registry/billing/PublishInvoicesActionTest.java b/javatests/google/registry/billing/PublishInvoicesActionTest.java new file mode 100644 index 000000000..db3aa736f --- /dev/null +++ b/javatests/google/registry/billing/PublishInvoicesActionTest.java @@ -0,0 +1,95 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.billing; + +import static com.google.common.truth.Truth.assertThat; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; +import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.Dataflow.Projects; +import com.google.api.services.dataflow.Dataflow.Projects.Jobs; +import com.google.api.services.dataflow.Dataflow.Projects.Jobs.Get; +import com.google.api.services.dataflow.model.Job; +import com.google.common.net.MediaType; +import google.registry.testing.FakeResponse; +import java.io.IOException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class PublishInvoicesActionTest { + + private final Dataflow dataflow = mock(Dataflow.class); + private final Projects projects = mock(Projects.class); + private final Jobs jobs = mock(Jobs.class); + private final Get get = mock(Get.class); + + private Job expectedJob; + private FakeResponse response; + private PublishInvoicesAction uploadAction; + + @Before + public void initializeObjects() throws Exception { + when(dataflow.projects()).thenReturn(projects); + when(projects.jobs()).thenReturn(jobs); + when(jobs.get("test-project", "12345")).thenReturn(get); + expectedJob = new Job(); + when(get.execute()).thenReturn(expectedJob); + + uploadAction = new PublishInvoicesAction(); + uploadAction.projectId = "test-project"; + uploadAction.jobId = "12345"; + uploadAction.dataflow = dataflow; + response = new FakeResponse(); + uploadAction.response = response; + } + + @Test + public void testJobDone_returnsSuccess() { + expectedJob.setCurrentState("JOB_STATE_DONE"); + uploadAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_OK); + } + + @Test + public void testJobFailed_returnsNonRetriableResponse() { + expectedJob.setCurrentState("JOB_STATE_FAILED"); + uploadAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT); + } + + @Test + public void testJobIndeterminate_returnsRetriableResponse() { + expectedJob.setCurrentState("JOB_STATE_RUNNING"); + uploadAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_NOT_MODIFIED); + } + + @Test + public void testIOException_returnsFailureMessage() throws Exception { + when(get.execute()).thenThrow(new IOException("expected")); + uploadAction.run(); + assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR); + assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8); + assertThat(response.getPayload()).isEqualTo("Template launch failed: expected"); + } +} diff --git a/javatests/google/registry/module/backend/BUILD b/javatests/google/registry/module/backend/BUILD index aacb6920f..164349865 100644 --- a/javatests/google/registry/module/backend/BUILD +++ b/javatests/google/registry/module/backend/BUILD @@ -22,6 +22,7 @@ java_library( "@com_google_truth", "@com_google_truth_extensions_truth_java8_extension", "@javax_servlet_api", + "@joda_time", "@junit", "@org_mockito_all", ], diff --git a/javatests/google/registry/module/backend/BackendModuleTest.java b/javatests/google/registry/module/backend/BackendModuleTest.java new file mode 100644 index 000000000..0abb8cbfc --- /dev/null +++ b/javatests/google/registry/module/backend/BackendModuleTest.java @@ -0,0 +1,74 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.module.backend; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.testing.JUnitBackports.expectThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import google.registry.request.HttpException.BadRequestException; +import google.registry.testing.FakeClock; +import google.registry.util.Clock; +import java.util.Optional; +import javax.servlet.http.HttpServletRequest; +import org.joda.time.DateTime; +import org.joda.time.YearMonth; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link BackendModule}. */ +@RunWith(JUnit4.class) +public class BackendModuleTest { + + private HttpServletRequest req = mock(HttpServletRequest.class); + private Clock clock; + @Before + public void setUp() { + clock = new FakeClock(DateTime.parse("2017-07-01TZ")); + } + + @Test + public void testEmptyYearMonthParameter_returnsEmptyYearMonthOptional() { + when(req.getParameter("yearMonth")).thenReturn(""); + assertThat(BackendModule.provideYearMonthOptional(req)).isEqualTo(Optional.empty()); + } + + @Test + public void testInvalidYearMonthParameter_throwsException() { + when(req.getParameter("yearMonth")).thenReturn("201705"); + BadRequestException thrown = + expectThrows( + BadRequestException.class, () -> BackendModule.provideYearMonthOptional(req)); + assertThat(thrown) + .hasMessageThat() + .contains("yearMonth must be in yyyy-MM format, got 201705 instead"); + } + + @Test + public void testEmptyYearMonth_returnsLastMonth() { + assertThat(BackendModule.provideYearMonth(Optional.empty(), clock)) + .isEqualTo(new YearMonth(2017, 6)); + } + + @Test + public void testGivenYearMonth_returnsThatMonth() { + assertThat(BackendModule.provideYearMonth(Optional.of(new YearMonth(2017, 5)), clock)) + .isEqualTo(new YearMonth(2017, 5)); + } + +} diff --git a/javatests/google/registry/module/backend/testdata/backend_routing.txt b/javatests/google/registry/module/backend/testdata/backend_routing.txt index 5f01080e6..2db9cce71 100644 --- a/javatests/google/registry/module/backend/testdata/backend_routing.txt +++ b/javatests/google/registry/module/backend/testdata/backend_routing.txt @@ -28,6 +28,7 @@ PATH CLASS METHOD /_dr/task/nordnVerify NordnVerifyAction POST y INTERNAL APP IGNORED /_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED /_dr/task/publishDnsUpdates PublishDnsUpdatesAction POST y INTERNAL APP IGNORED +/_dr/task/publishInvoices PublishInvoicesAction POST n INTERNAL,API APP ADMIN /_dr/task/rdeReport RdeReportAction POST n INTERNAL APP IGNORED /_dr/task/rdeStaging RdeStagingAction GET,POST n INTERNAL APP IGNORED /_dr/task/rdeUpload RdeUploadAction POST n INTERNAL APP IGNORED diff --git a/javatests/google/registry/reporting/IcannReportingModuleTest.java b/javatests/google/registry/reporting/IcannReportingModuleTest.java index c1a78dc73..acc633662 100644 --- a/javatests/google/registry/reporting/IcannReportingModuleTest.java +++ b/javatests/google/registry/reporting/IcannReportingModuleTest.java @@ -16,18 +16,11 @@ package google.registry.reporting; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.JUnitBackports.expectThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import google.registry.reporting.IcannReportingModule.ReportType; import google.registry.request.HttpException.BadRequestException; -import google.registry.testing.FakeClock; -import google.registry.util.Clock; import java.util.Optional; -import javax.servlet.http.HttpServletRequest; -import org.joda.time.DateTime; import org.joda.time.YearMonth; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -36,42 +29,6 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class IcannReportingModuleTest { - HttpServletRequest req = mock(HttpServletRequest.class); - Clock clock; - @Before - public void setUp() { - clock = new FakeClock(DateTime.parse("2017-07-01TZ")); - } - - @Test - public void testEmptyYearMonthParameter_returnsEmptyYearMonthOptional() { - when(req.getParameter("yearMonth")).thenReturn(""); - assertThat(IcannReportingModule.provideYearMonthOptional(req)).isEqualTo(Optional.empty()); - } - - @Test - public void testInvalidYearMonthParameter_throwsException() { - when(req.getParameter("yearMonth")).thenReturn("201705"); - BadRequestException thrown = - expectThrows( - BadRequestException.class, () -> IcannReportingModule.provideYearMonthOptional(req)); - assertThat(thrown) - .hasMessageThat() - .contains("yearMonth must be in yyyy-MM format, got 201705 instead"); - } - - @Test - public void testEmptyYearMonth_returnsLastMonth() { - assertThat(IcannReportingModule.provideYearMonth(Optional.empty(), clock)) - .isEqualTo(new YearMonth(2017, 6)); - } - - @Test - public void testGivenYearMonth_returnsThatMonth() { - assertThat(IcannReportingModule.provideYearMonth(Optional.of(new YearMonth(2017, 5)), clock)) - .isEqualTo(new YearMonth(2017, 5)); - } - @Test public void testEmptySubDir_returnsDefaultSubdir() { assertThat(IcannReportingModule.provideSubdir(Optional.empty(), new YearMonth(2017, 6))) diff --git a/javatests/google/registry/reporting/IcannReportingStagingActionTest.java b/javatests/google/registry/reporting/IcannReportingStagingActionTest.java index b77485ea2..8b2e4febc 100644 --- a/javatests/google/registry/reporting/IcannReportingStagingActionTest.java +++ b/javatests/google/registry/reporting/IcannReportingStagingActionTest.java @@ -62,12 +62,11 @@ public class IcannReportingStagingActionTest { when(stager.stageReports(ReportType.TRANSACTIONS)).thenReturn(ImmutableList.of("c", "d")); } - private static void assertUploadTaskEnqueued(String yearMonth, String subDir) throws Exception { + private static void assertUploadTaskEnqueued(String subDir) throws Exception { TaskMatcher matcher = new TaskMatcher() .url("/_dr/task/icannReportingUpload") .method("POST") - .param("yearMonth", yearMonth) .param("subdir", subDir); assertTasksEnqueued("retryable-cron-tasks", matcher); } @@ -94,7 +93,7 @@ public class IcannReportingStagingActionTest { .emailResults( "ICANN Monthly report staging summary [SUCCESS]", "Completed staging the following 2 ICANN reports:\na\nb"); - assertUploadTaskEnqueued("2017-06", "default/dir"); + assertUploadTaskEnqueued("default/dir"); } @Test @@ -109,7 +108,7 @@ public class IcannReportingStagingActionTest { .emailResults( "ICANN Monthly report staging summary [SUCCESS]", "Completed staging the following 4 ICANN reports:\na\nb\nc\nd"); - assertUploadTaskEnqueued("2017-06", "default/dir"); + assertUploadTaskEnqueued("default/dir"); } @Test @@ -127,7 +126,7 @@ public class IcannReportingStagingActionTest { .emailResults( "ICANN Monthly report staging summary [SUCCESS]", "Completed staging the following 4 ICANN reports:\na\nb\nc\nd"); - assertUploadTaskEnqueued("2017-06", "default/dir"); + assertUploadTaskEnqueued("default/dir"); } @Test