diff --git a/core/src/main/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java b/core/src/main/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java index 18524a19e..6374a244c 100644 --- a/core/src/main/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java +++ b/core/src/main/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java @@ -82,7 +82,6 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder { // Convert reportingMonth into YYYYMMDD format for Bigquery table partition pattern-matching. DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd"); - // The monthly logs are a shared dependency for epp counts and whois metrics String monthlyLogsQuery = SqlTemplate.create(getQueryFromFile("monthly_logs.sql")) .put("PROJECT_ID", projectId) @@ -96,12 +95,10 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder { String eppQuery = SqlTemplate.create(getQueryFromFile("epp_metrics.sql")) .put("PROJECT_ID", projectId) - .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet) - .put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS, yearMonth)) - // All metadata logs for reporting come from google.registry.flows.FlowReporter. - .put( - "METADATA_LOG_PREFIX", - "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA") + .put("APPENGINE_LOGS_DATA_SET", "appengine_logs") + .put("APP_LOGS_TABLE", "_var_log_app_") + .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(firstDayOfMonth)) + .put("LAST_DAY_OF_MONTH", logTableFormatter.print(lastDayOfMonth)) .build(); queriesBuilder.put(getTableName(EPP_METRICS, yearMonth), eppQuery); diff --git a/core/src/main/java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java b/core/src/main/java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java index 4a449dbec..4f36e5831 100644 --- a/core/src/main/java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java +++ b/core/src/main/java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java @@ -113,13 +113,9 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder { SqlTemplate.create(getQueryFromFile("cloud_sql_attempted_adds.sql")) .put("PROJECT_ID", projectId) .put("APPENGINE_LOGS_DATA_SET", "appengine_logs") - .put("REQUEST_TABLE", "appengine_googleapis_com_request_log_") + .put("APP_LOGS_TABLE", "_var_log_app_") .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime)) .put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime)) - // All metadata logs for reporting come from google.registry.flows.FlowReporter. - .put( - "METADATA_LOG_PREFIX", - "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA") .build(); queriesBuilder.put(getTableName(ATTEMPTED_ADDS, yearMonth), attemptedAddsQuery); diff --git a/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_attempted_adds.sql b/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_attempted_adds.sql index 385e85a9d..d32315254 100644 --- a/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_attempted_adds.sql +++ b/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_attempted_adds.sql @@ -41,29 +41,23 @@ FROM ( FROM ( -- Extract JSON metadata package from monthly logs SELECT - REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') + REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json FROM ( SELECT - protoPayload.resource AS requestPath, - ARRAY( - SELECT logMessage - FROM UNNEST(protoPayload.line)) AS logMessage + textPayload FROM - `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*` + `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%APP_LOGS_TABLE%*` WHERE _TABLE_SUFFIX BETWEEN '%FIRST_DAY_OF_MONTH%' AND '%LAST_DAY_OF_MONTH%') - JOIN UNNEST(logMessage) AS logMessages - -- Look for metadata logs from epp and registrar console requests - WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr') - AND STARTS_WITH(logMessages, "%METADATA_LOG_PREFIX%") + WHERE STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA") -- Look for domain creates AND REGEXP_CONTAINS( - logMessages, r'"commandType":"create","resourceType":"domain"') + textPayload, r'"commandType":"create","resourceType":"domain"') -- Filter prober data AND NOT REGEXP_CONTAINS( - logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') ) + textPayload, r'"prober-[a-z]{2}-((any)|(canary))"') ) GROUP BY tld, clientId ) AS logs_table JOIN EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql", diff --git a/core/src/main/resources/google/registry/reporting/icann/sql/epp_metrics.sql b/core/src/main/resources/google/registry/reporting/icann/sql/epp_metrics.sql index 91151fbf1..ced9a2b94 100644 --- a/core/src/main/resources/google/registry/reporting/icann/sql/epp_metrics.sql +++ b/core/src/main/resources/google/registry/reporting/icann/sql/epp_metrics.sql @@ -37,13 +37,12 @@ FROM ( FROM ( SELECT -- Extract the logged JSON payload. - REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') + REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json - FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%MONTHLY_LOGS_TABLE%` AS logs - JOIN - UNNEST(logs.logMessage) AS logMessage + FROM `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%APP_LOGS_TABLE%*` WHERE - STARTS_WITH(logMessage, "%METADATA_LOG_PREFIX%"))) AS regexes + STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA") + AND _TABLE_SUFFIX BETWEEN '%FIRST_DAY_OF_MONTH%' AND '%LAST_DAY_OF_MONTH%')) AS regexes JOIN -- Unnest the JSON-parsed tlds. UNNEST(regexes.tlds) AS tld diff --git a/core/src/main/resources/google/registry/reporting/icann/sql/monthly_logs.sql b/core/src/main/resources/google/registry/reporting/icann/sql/monthly_logs.sql index e47d56ec7..7570b8f95 100644 --- a/core/src/main/resources/google/registry/reporting/icann/sql/monthly_logs.sql +++ b/core/src/main/resources/google/registry/reporting/icann/sql/monthly_logs.sql @@ -19,11 +19,6 @@ SELECT protoPayload.resource AS requestPath, - ARRAY( - SELECT - logMessage - FROM - UNNEST(protoPayload.line)) AS logMessage FROM `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*` WHERE diff --git a/core/src/test/resources/google/registry/reporting/icann/attempted_adds_test_cloud_sql.sql b/core/src/test/resources/google/registry/reporting/icann/attempted_adds_test_cloud_sql.sql index 8c6645008..c40c766c7 100644 --- a/core/src/test/resources/google/registry/reporting/icann/attempted_adds_test_cloud_sql.sql +++ b/core/src/test/resources/google/registry/reporting/icann/attempted_adds_test_cloud_sql.sql @@ -41,29 +41,23 @@ FROM ( FROM ( -- Extract JSON metadata package from monthly logs SELECT - REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') + REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json FROM ( SELECT - protoPayload.resource AS requestPath, - ARRAY( - SELECT logMessage - FROM UNNEST(protoPayload.line)) AS logMessage + textPayload FROM - `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*` + `domain-registry-alpha.appengine_logs._var_log_app_*` WHERE _TABLE_SUFFIX BETWEEN '20170901' AND '20170930') - JOIN UNNEST(logMessage) AS logMessages - -- Look for metadata logs from epp and registrar console requests - WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr') - AND STARTS_WITH(logMessages, "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA") + WHERE STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA") -- Look for domain creates AND REGEXP_CONTAINS( - logMessages, r'"commandType":"create","resourceType":"domain"') + textPayload, r'"commandType":"create","resourceType":"domain"') -- Filter prober data AND NOT REGEXP_CONTAINS( - logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') ) + textPayload, r'"prober-[a-z]{2}-((any)|(canary))"') ) GROUP BY tld, clientId ) AS logs_table JOIN EXTERNAL_QUERY("projects/domain-registry-alpha/locations/us/connections/domain-registry-alpha-sql", diff --git a/core/src/test/resources/google/registry/reporting/icann/epp_metrics_test_cloud_sql.sql b/core/src/test/resources/google/registry/reporting/icann/epp_metrics_test_cloud_sql.sql index 151cc1728..ae0424894 100644 --- a/core/src/test/resources/google/registry/reporting/icann/epp_metrics_test_cloud_sql.sql +++ b/core/src/test/resources/google/registry/reporting/icann/epp_metrics_test_cloud_sql.sql @@ -37,13 +37,12 @@ FROM ( FROM ( SELECT -- Extract the logged JSON payload. - REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') + REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json - FROM `domain-registry-alpha.cloud_sql_icann_reporting.monthly_logs_201709` AS logs - JOIN - UNNEST(logs.logMessage) AS logMessage + FROM `domain-registry-alpha.appengine_logs._var_log_app_*` WHERE - STARTS_WITH(logMessage, "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA"))) AS regexes + STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA") + AND _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')) AS regexes JOIN -- Unnest the JSON-parsed tlds. UNNEST(regexes.tlds) AS tld diff --git a/core/src/test/resources/google/registry/reporting/icann/monthly_logs_test_cloud_sql.sql b/core/src/test/resources/google/registry/reporting/icann/monthly_logs_test_cloud_sql.sql index 86660f304..ae61bb2a0 100644 --- a/core/src/test/resources/google/registry/reporting/icann/monthly_logs_test_cloud_sql.sql +++ b/core/src/test/resources/google/registry/reporting/icann/monthly_logs_test_cloud_sql.sql @@ -19,11 +19,6 @@ SELECT protoPayload.resource AS requestPath, - ARRAY( - SELECT - logMessage - FROM - UNNEST(protoPayload.line)) AS logMessage FROM `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*` WHERE