Update activity/txn reporting to use new GAE log format (#2535)

Instead of having to parse the protoPayload.line from the request logs, we just want to inspect the textPayload from the app logs (stored in a separate table). This applies to the EPP metrics from the activity reporting and the attempted-adds column for the transaction reporting.
2026-01-03 11:45:39 +00:00 · 2024-08-26 15:41:40 -04:00
parent 1739c6d74f
commit e88c6e1550
8 changed files with 25 additions and 56 deletions
--- a/core/src/main/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java
+++ b/core/src/main/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java
@@ -82,7 +82,6 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {

    // Convert reportingMonth into YYYYMMDD format for Bigquery table partition pattern-matching.
    DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd");
-    // The monthly logs are a shared dependency for epp counts and whois metrics
    String monthlyLogsQuery =
        SqlTemplate.create(getQueryFromFile("monthly_logs.sql"))
            .put("PROJECT_ID", projectId)
@@ -96,12 +95,10 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
    String eppQuery =
        SqlTemplate.create(getQueryFromFile("epp_metrics.sql"))
            .put("PROJECT_ID", projectId)
-            .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
-            .put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS, yearMonth))
-            // All metadata logs for reporting come from google.registry.flows.FlowReporter.
-            .put(
-                "METADATA_LOG_PREFIX",
-                "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
+            .put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
+            .put("APP_LOGS_TABLE", "_var_log_app_")
+            .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(firstDayOfMonth))
+            .put("LAST_DAY_OF_MONTH", logTableFormatter.print(lastDayOfMonth))
            .build();
    queriesBuilder.put(getTableName(EPP_METRICS, yearMonth), eppQuery);

--- a/core/src/main/java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java
+++ b/core/src/main/java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java
@@ -113,13 +113,9 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
        SqlTemplate.create(getQueryFromFile("cloud_sql_attempted_adds.sql"))
            .put("PROJECT_ID", projectId)
            .put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
-            .put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
+            .put("APP_LOGS_TABLE", "_var_log_app_")
            .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
            .put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
-            // All metadata logs for reporting come from google.registry.flows.FlowReporter.
-            .put(
-                "METADATA_LOG_PREFIX",
-                "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
            .build();
    queriesBuilder.put(getTableName(ATTEMPTED_ADDS, yearMonth), attemptedAddsQuery);

--- a/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_attempted_adds.sql
+++ b/core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_attempted_adds.sql
@@ -41,29 +41,23 @@ FROM (
  FROM (
      -- Extract JSON metadata package from monthly logs
    SELECT
-      REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+      REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
          AS json
    FROM (
      SELECT
-        protoPayload.resource AS requestPath,
-        ARRAY(
-          SELECT logMessage
-          FROM UNNEST(protoPayload.line)) AS logMessage
+        textPayload
      FROM
-        `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*`
+        `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%APP_LOGS_TABLE%*`
      WHERE _TABLE_SUFFIX
      BETWEEN '%FIRST_DAY_OF_MONTH%'
      AND '%LAST_DAY_OF_MONTH%')
-    JOIN UNNEST(logMessage) AS logMessages
-    -- Look for metadata logs from epp and registrar console requests
-    WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
-    AND STARTS_WITH(logMessages, "%METADATA_LOG_PREFIX%")
+    WHERE STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
    -- Look for domain creates
    AND REGEXP_CONTAINS(
-       logMessages, r'"commandType":"create","resourceType":"domain"')
+       textPayload, r'"commandType":"create","resourceType":"domain"')
    -- Filter prober data
    AND NOT REGEXP_CONTAINS(
-        logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') )
+        textPayload, r'"prober-[a-z]{2}-((any)|(canary))"') )
  GROUP BY tld, clientId ) AS logs_table
 JOIN
  EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
--- a/core/src/main/resources/google/registry/reporting/icann/sql/epp_metrics.sql
+++ b/core/src/main/resources/google/registry/reporting/icann/sql/epp_metrics.sql
@@ -37,13 +37,12 @@ FROM (
  FROM (
    SELECT
      -- Extract the logged JSON payload.
-      REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+      REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
      AS json
-    FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%MONTHLY_LOGS_TABLE%` AS logs
-    JOIN
-      UNNEST(logs.logMessage) AS logMessage
+    FROM `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%APP_LOGS_TABLE%*`
    WHERE
-      STARTS_WITH(logMessage, "%METADATA_LOG_PREFIX%"))) AS regexes
+      STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
+      AND _TABLE_SUFFIX BETWEEN '%FIRST_DAY_OF_MONTH%' AND '%LAST_DAY_OF_MONTH%')) AS regexes
 JOIN
  -- Unnest the JSON-parsed tlds.
  UNNEST(regexes.tlds) AS tld
--- a/core/src/main/resources/google/registry/reporting/icann/sql/monthly_logs.sql
+++ b/core/src/main/resources/google/registry/reporting/icann/sql/monthly_logs.sql
@@ -19,11 +19,6 @@

 SELECT
  protoPayload.resource AS requestPath,
-  ARRAY(
-  SELECT
-    logMessage
-  FROM
-    UNNEST(protoPayload.line)) AS logMessage
 FROM
  `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*`
 WHERE
--- a/core/src/test/resources/google/registry/reporting/icann/attempted_adds_test_cloud_sql.sql
+++ b/core/src/test/resources/google/registry/reporting/icann/attempted_adds_test_cloud_sql.sql
@@ -41,29 +41,23 @@ FROM (
  FROM (
      -- Extract JSON metadata package from monthly logs
    SELECT
-      REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+      REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
          AS json
    FROM (
      SELECT
-        protoPayload.resource AS requestPath,
-        ARRAY(
-          SELECT logMessage
-          FROM UNNEST(protoPayload.line)) AS logMessage
+        textPayload
      FROM
-        `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
+        `domain-registry-alpha.appengine_logs._var_log_app_*`
      WHERE _TABLE_SUFFIX
      BETWEEN '20170901'
      AND '20170930')
-    JOIN UNNEST(logMessage) AS logMessages
-    -- Look for metadata logs from epp and registrar console requests
-    WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
-    AND STARTS_WITH(logMessages, "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
+    WHERE STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
    -- Look for domain creates
    AND REGEXP_CONTAINS(
-       logMessages, r'"commandType":"create","resourceType":"domain"')
+       textPayload, r'"commandType":"create","resourceType":"domain"')
    -- Filter prober data
    AND NOT REGEXP_CONTAINS(
-        logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') )
+        textPayload, r'"prober-[a-z]{2}-((any)|(canary))"') )
  GROUP BY tld, clientId ) AS logs_table
 JOIN
  EXTERNAL_QUERY("projects/domain-registry-alpha/locations/us/connections/domain-registry-alpha-sql",
--- a/core/src/test/resources/google/registry/reporting/icann/epp_metrics_test_cloud_sql.sql
+++ b/core/src/test/resources/google/registry/reporting/icann/epp_metrics_test_cloud_sql.sql
@@ -37,13 +37,12 @@ FROM (
  FROM (
    SELECT
      -- Extract the logged JSON payload.
-      REGEXP_EXTRACT(logMessage, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+      REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
      AS json
-    FROM `domain-registry-alpha.cloud_sql_icann_reporting.monthly_logs_201709` AS logs
-    JOIN
-      UNNEST(logs.logMessage) AS logMessage
+    FROM `domain-registry-alpha.appengine_logs._var_log_app_*`
    WHERE
-      STARTS_WITH(logMessage, "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA"))) AS regexes
+      STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
+      AND _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')) AS regexes
 JOIN
  -- Unnest the JSON-parsed tlds.
  UNNEST(regexes.tlds) AS tld
--- a/core/src/test/resources/google/registry/reporting/icann/monthly_logs_test_cloud_sql.sql
+++ b/core/src/test/resources/google/registry/reporting/icann/monthly_logs_test_cloud_sql.sql
@@ -19,11 +19,6 @@

 SELECT
  protoPayload.resource AS requestPath,
-  ARRAY(
-  SELECT
-    logMessage
-  FROM
-    UNNEST(protoPayload.line)) AS logMessage
 FROM
  `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
 WHERE