mirror of
https://github.com/google/nomulus
synced 2026-09-03 22:57:09 +00:00
Migrate ICANN activity reports to Cloud SQL on BQ (#1332)
* Migrate ICANN activity reports to Cloud SQL on BQ * Fix data set name
This commit is contained in:
sarahcaseybot
+37
-18
@@ -14,8 +14,8 @@
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
|
||||
import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
|
||||
import static google.registry.reporting.icann.QueryBuilderUtils.getQueryFromFile;
|
||||
import static google.registry.reporting.icann.QueryBuilderUtils.getTableName;
|
||||
|
||||
@@ -38,6 +38,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||
static final String EPP_METRICS = "epp_metrics";
|
||||
static final String WHOIS_COUNTS = "whois_counts";
|
||||
static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
|
||||
final String BIGQUERY_DATA_SET = tm().isOfy() ? "icann_reporting" : "cloud_sql_icann_reporting";
|
||||
|
||||
@Inject
|
||||
@Config("projectId")
|
||||
@@ -53,7 +54,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||
public String getReportQuery(YearMonth yearMonth) {
|
||||
return String.format(
|
||||
"#standardSQL\nSELECT * FROM `%s.%s.%s`",
|
||||
projectId, ICANN_REPORTING_DATA_SET, getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth));
|
||||
projectId, BIGQUERY_DATA_SET, getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth));
|
||||
}
|
||||
|
||||
/** Sets the month we're doing activity reporting for, and returns the view query map. */
|
||||
@@ -65,12 +66,20 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||
LocalDate lastDayOfMonth = yearMonth.toLocalDate(1).plusMonths(1).minusDays(1);
|
||||
|
||||
ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
|
||||
String operationalRegistrarsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("registrar_operating_status.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.build();
|
||||
String operationalRegistrarsQuery;
|
||||
if (tm().isOfy()) {
|
||||
operationalRegistrarsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("registrar_operating_status.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.build();
|
||||
} else {
|
||||
operationalRegistrarsQuery =
|
||||
SqlTemplate.create(getQueryFromFile("cloud_sql_registrar_operating_status.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.build();
|
||||
}
|
||||
queriesBuilder.put(
|
||||
getTableName(REGISTRAR_OPERATING_STATUS, yearMonth), operationalRegistrarsQuery);
|
||||
|
||||
@@ -93,7 +102,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||
String eppQuery =
|
||||
SqlTemplate.create(getQueryFromFile("epp_metrics.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("ICANN_REPORTING_DATA_SET", BIGQUERY_DATA_SET)
|
||||
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS, yearMonth))
|
||||
// All metadata logs for reporting come from google.registry.flows.FlowReporter.
|
||||
.put(
|
||||
@@ -105,25 +114,35 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
|
||||
String whoisQuery =
|
||||
SqlTemplate.create(getQueryFromFile("whois_counts.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put("ICANN_REPORTING_DATA_SET", BIGQUERY_DATA_SET)
|
||||
.put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS, yearMonth))
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(WHOIS_COUNTS, yearMonth), whoisQuery);
|
||||
|
||||
String aggregateQuery =
|
||||
SqlTemplate.create(getQueryFromFile("activity_report_aggregation.sql"))
|
||||
SqlTemplate aggregateQuery =
|
||||
SqlTemplate.create(
|
||||
getQueryFromFile(
|
||||
tm().isOfy()
|
||||
? "activity_report_aggregation.sql"
|
||||
: "cloud_sql_activity_report_aggregation.sql"))
|
||||
.put("PROJECT_ID", projectId)
|
||||
.put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
|
||||
.put(
|
||||
"REGISTRAR_OPERATING_STATUS_TABLE",
|
||||
getTableName(REGISTRAR_OPERATING_STATUS, yearMonth))
|
||||
.put("ICANN_REPORTING_DATA_SET", BIGQUERY_DATA_SET)
|
||||
.put("DNS_COUNTS_TABLE", getTableName(DNS_COUNTS, yearMonth))
|
||||
.put("EPP_METRICS_TABLE", getTableName(EPP_METRICS, yearMonth))
|
||||
.put("WHOIS_COUNTS_TABLE", getTableName(WHOIS_COUNTS, yearMonth))
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
|
||||
.put("REGISTRY_TABLE", "Registry")
|
||||
.build();
|
||||
queriesBuilder.put(getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth), aggregateQuery);
|
||||
.put("WHOIS_COUNTS_TABLE", getTableName(WHOIS_COUNTS, yearMonth));
|
||||
|
||||
if (tm().isOfy()) {
|
||||
aggregateQuery =
|
||||
aggregateQuery
|
||||
.put("REGISTRY_TABLE", "Registry")
|
||||
.put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET);
|
||||
}
|
||||
|
||||
queriesBuilder.put(
|
||||
getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth), aggregateQuery.build());
|
||||
|
||||
return queriesBuilder.build();
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static google.registry.request.RequestParameters.extractSetOfEnumParameters;
|
||||
@@ -41,7 +42,8 @@ public final class IcannReportingModule {
|
||||
|
||||
static final String PARAM_SUBDIR = "subdir";
|
||||
static final String PARAM_REPORT_TYPES = "reportTypes";
|
||||
static final String ICANN_REPORTING_DATA_SET = "icann_reporting";
|
||||
static final String ICANN_REPORTING_DATA_SET =
|
||||
tm().isOfy() ? "icann_reporting" : "cloud_sql_icann_reporting";
|
||||
static final String DATASTORE_EXPORT_DATA_SET = "latest_datastore_export";
|
||||
static final String MANIFEST_FILE_NAME = "MANIFEST.txt";
|
||||
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
#standardSQL
|
||||
-- Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- This query pulls from all intermediary tables to create the activity
|
||||
-- report csv, via a table transpose and sum over all activity report fields.
|
||||
|
||||
SELECT
|
||||
RealTlds.tld AS tld,
|
||||
SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
|
||||
-- We use the Centralized Zone Data Service.
|
||||
"CZDS" AS zfa_passwords,
|
||||
SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
|
||||
SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries,
|
||||
-- We don't support searchable WHOIS.
|
||||
0 AS searchable_whois_queries,
|
||||
-- DNS queries for UDP/TCP are all assumed to be received/responded.
|
||||
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received,
|
||||
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded,
|
||||
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received,
|
||||
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded,
|
||||
-- SRS metrics.
|
||||
SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check,
|
||||
SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create,
|
||||
SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete,
|
||||
SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info,
|
||||
SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew,
|
||||
SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report,
|
||||
SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject,
|
||||
SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request,
|
||||
SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update,
|
||||
SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check,
|
||||
SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create,
|
||||
SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete,
|
||||
SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info,
|
||||
SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update,
|
||||
SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check,
|
||||
SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create,
|
||||
SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete,
|
||||
SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject,
|
||||
SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request,
|
||||
SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update
|
||||
-- Cross join a list of all TLDs against TLD-specific metrics and then
|
||||
-- filter so that only metrics with that TLD or a NULL TLD are counted
|
||||
-- towards a given TLD.
|
||||
FROM
|
||||
EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
|
||||
'''SELECT
|
||||
"tld_name" AS tld
|
||||
FROM (
|
||||
SELECT
|
||||
"tld_name"
|
||||
FROM
|
||||
"Tld" AS t
|
||||
WHERE t.tld_type='REAL'
|
||||
) AS "RealTlds";''') AS RealTlds
|
||||
CROSS JOIN(
|
||||
SELECT
|
||||
tld,
|
||||
metricName,
|
||||
count
|
||||
FROM
|
||||
(
|
||||
-- BEGIN INTERMEDIARY DATA SOURCES --
|
||||
-- Dummy data source to ensure all TLDs appear in report, even if
|
||||
-- they have no recorded metrics for the month.
|
||||
SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 as count
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_OPERATING_STATUS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%DNS_COUNTS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%EPP_METRICS_TABLE%`
|
||||
UNION ALL
|
||||
SELECT * FROM
|
||||
`%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%WHOIS_COUNTS_TABLE%`
|
||||
-- END INTERMEDIARY DATA SOURCES --
|
||||
)) AS TldMetrics
|
||||
WHERE RealTlds.tld = TldMetrics.tld OR TldMetrics.tld IS NULL
|
||||
GROUP BY tld
|
||||
ORDER BY tld
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
#standardSQL
|
||||
-- Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- Query that counts the number of real registrars in system.
|
||||
|
||||
SELECT
|
||||
-- Applies to all TLDs, hence the 'null' magic value.
|
||||
STRING(NULL) AS tld,
|
||||
'operational-registrars' AS metricName,
|
||||
count
|
||||
FROM
|
||||
EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
|
||||
'''SELECT
|
||||
COUNT("registrar_name") AS count
|
||||
FROM
|
||||
"Registrar" AS r
|
||||
WHERE
|
||||
r.type='REAL' OR r.type='INTERNAL' ;''')
|
||||
GROUP BY count
|
||||
Reference in New Issue
Block a user