mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-07-31 04:22:47 +00:00
Code Issues Releases Wiki Activity

Update input/output of Spec11 pipeline to final format

Browse Source 
This changes the BigQuery input to the fields we ultimately want (fqdn,
registrarName, registrarEmailAddress) and the output to a structured POJO
holding the results from the API. This POJO is then converted to its final text output, i.e.:

Map from registrar e-mail to list of threat-detected subdomains:
{"registrarEmail": "c@fake.com", "threats": [{"url": "a.com", "threatType": "MALWARE"}]}
{"registrarEmail": "d@fake.com", "threats": [{"url": "x.com", "threatType": "MALWARE"}, {"url": "y.com", "threatType": "MALWARE"}]}

This gives us all the data we want in a JSON structured format, to be acted upon downstream by the to-be-constructed PublishSpec11ReportAction. Ideally, we would send an e-mail directly from the beam pipeline, but this is only possible through third-party providers (as opposed to app engine itself).

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=209416880
This commit is contained in:
larryruili
2018-08-20 14:26:46 -04:00
committed by jianglai
parent 7dcadaecf6
commit f7bc17fbe8
11 changed files with 393 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/google/registry/beam/invoicing/InvoicingUtils.java
+3 -9
View File
@@ -14,8 +14,8 @@
package google.registry.beam.invoicing;
import com.google.common.io.Resources;
import google.registry.util.ResourceUtils;
import static google.registry.beam.BeamUtils.getQueryFromFile;
import google.registry.util.SqlTemplate;
import java.time.LocalDateTime;
import java.time.LocalTime;
@@ -91,7 +91,7 @@ public class InvoicingUtils {
LocalDateTime firstMoment = reportingMonth.atDay(1).atTime(LocalTime.MIDNIGHT);
LocalDateTime lastMoment = reportingMonth.atEndOfMonth().atTime(LocalTime.MAX);
// Construct the month's query by filling in the billing_events.sql template
return SqlTemplate.create(getQueryFromFile("billing_events.sql"))
return SqlTemplate.create(getQueryFromFile(InvoicingPipeline.class, "billing_events.sql"))
.put("FIRST_TIMESTAMP_OF_MONTH", firstMoment.format(TIMESTAMP_FORMATTER))
.put("LAST_TIMESTAMP_OF_MONTH", lastMoment.format(TIMESTAMP_FORMATTER))
.put("PROJECT_ID", projectId)
@@ -103,10 +103,4 @@ public class InvoicingUtils {
.build();
});
}
/** Returns the {@link String} contents for a file in the {@code beam/sql/} directory. */
private static String getQueryFromFile(String filename) {
return ResourceUtils.readResourceUtf8(
Resources.getResource(InvoicingUtils.class, "sql/" + filename));
}
}