Only use GKE logs in ICANN reports (#2738)

We no longer need to union GKE+GAE logs since we've moved all production
traffic to GKE only.

For testing, I copied the affected *_test.sql files to Bigquery, removed
all the "-alpha" bits, and changed the dates to 20250301 and 20250331
and ran them to make sure they returned the expected data.

core/src/test/resources/google/registry/reporting/icann/attempted_adds_test.sql

@@ -16,7 +16,7 @@
   -- Determine the number of attempted adds each registrar made.
 
   -- Since the specification requests all 'attempted' adds, we regex the
-  -- monthly App Engine logs, searching for all create commands and associating
+  -- monthly GKE logs, searching for all create commands and associating
   -- them with their corresponding registrars.
 
   -- Example log generated by FlowReporter in App Engine and GKE logs:
@@ -27,7 +27,7 @@
   -- ,"targetId":"","targetIds":[],"tld":"",
   -- "tlds":[],"icannActivityReportField":""}
 
--- This outer select just converts the registrar's clientId to their name.
+-- This outer select just converts the registrar's ID to their name.
 SELECT
   tld,
   registrar_table.registrar_name AS registrar_name,
@@ -38,34 +38,20 @@ FROM (
     JSON_EXTRACT_SCALAR(json, '$.tld') AS tld,
     JSON_EXTRACT_SCALAR(json, '$.clientId') AS clientId,
     COUNT(json) AS count
-  FROM ((
-      -- Extract JSON metadata package from monthly logs
+  FROM (
+    -- Extract JSON metadata package from monthly logs
     SELECT
-      REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json
+      REGEXP_EXTRACT(jsonPayload.message, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json
     FROM
-      `domain-registry-alpha.appengine_logs._var_log_app_*`
+      `domain-registry-alpha.gke_logs.stderr_*`
     WHERE
       _TABLE_SUFFIX BETWEEN '20170901'
       AND '20170930'
-      AND STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
+      AND STARTS_WITH(jsonPayload.message, "FLOW-LOG-SIGNATURE-METADATA")
       -- Look for domain creates
-      AND REGEXP_CONTAINS(textPayload, r'"commandType":"create","resourceType":"domain"')
+      AND REGEXP_CONTAINS(jsonPayload.message, r'"commandType":"create","resourceType":"domain"')
       -- Filter prober data
-      AND NOT REGEXP_CONTAINS(textPayload, r'"prober-[a-z]{2}-((any)|(canary))"'))
-    UNION ALL (
-        -- Extract JSON metadata package from monthly logs
-      SELECT
-        REGEXP_EXTRACT(jsonPayload.message, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json
-      FROM
-        `domain-registry-alpha.gke_logs.stderr_*`
-      WHERE
-        _TABLE_SUFFIX BETWEEN '20170901'
-        AND '20170930'
-        AND STARTS_WITH(jsonPayload.message, "FLOW-LOG-SIGNATURE-METADATA")
-        -- Look for domain creates
-        AND REGEXP_CONTAINS(jsonPayload.message, r'"commandType":"create","resourceType":"domain"')
-        -- Filter prober data
-        AND NOT REGEXP_CONTAINS(jsonPayload.message, r'"prober-[a-z]{2}-((any)|(canary))"')))
+      AND NOT REGEXP_CONTAINS(jsonPayload.message, r'"prober-[a-z]{2}-((any)|(canary))"'))
   GROUP BY
     tld,
     clientId ) AS logs_table

core/src/test/resources/google/registry/reporting/icann/epp_metrics_test.sql

@@ -15,7 +15,7 @@
 
   -- Query FlowReporter JSON log messages and calculate SRS metrics.
 
-  -- We use ugly regex's over the monthly appengine logs to determine how many
+  -- We use ugly regexes over the monthly GKE logs to determine how many
   -- EPP requests we received for each command. For example:
   -- {"commandType":"check"...,"targetIds":["ais.a.how"],
   -- "tld":"","tlds":["a.how"],"icannActivityReportField":"srs-dom-check"}
@@ -35,30 +35,15 @@ FROM (
     JSON_EXTRACT_SCALAR(json,
       '$.icannActivityReportField') AS activityReportField
   FROM (
-    -- For reasons that I don't understand, if I directly select the three columns
-    -- from the union, BigQuery complains about column number mismatch, so I have to
-    -- make a temporary union table and select on it.
     SELECT
-      *
-    FROM (
-      SELECT
-        -- Extract the logged JSON payload.
-        REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
-        AS json
-      FROM `domain-registry-alpha.appengine_logs._var_log_app_*`
-      WHERE
-        STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
-        AND _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')
-    UNION ALL (
-      SELECT
-        -- Extract the logged JSON payload.
-        REGEXP_EXTRACT(jsonPayload.message, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
-        AS json
-      FROM `domain-registry-alpha.gke_logs.stderr_*`
-      WHERE
-        STARTS_WITH(jsonPayload.message, "FLOW-LOG-SIGNATURE-METADATA")
-        AND _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')
-  )) AS regexes
+      -- Extract the logged JSON payload.
+      REGEXP_EXTRACT(jsonPayload.message, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+      AS json
+    FROM `domain-registry-alpha.gke_logs.stderr_*`
+    WHERE
+      STARTS_WITH(jsonPayload.message, "FLOW-LOG-SIGNATURE-METADATA")
+      AND _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')
+  ) AS regexes
 JOIN
   -- Unnest the JSON-parsed tlds.
   UNNEST(regexes.tlds) AS tld

core/src/test/resources/google/registry/reporting/icann/monthly_logs_test.sql

@@ -13,7 +13,7 @@
   -- See the License for the specific language governing permissions and
   -- limitations under the License.
 
-  -- Query to fetch AppEngine and GKE request logs for the report month.
+  -- Query to fetch GKE request logs for the report month.
 
   -- START_OF_MONTH and END_OF_MONTH should be in YYYYMM01 format.
 
@@ -26,10 +26,3 @@ FROM (
     `domain-registry-alpha.gke_logs.stderr_*`
   WHERE
     _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')
-UNION ALL (
-  SELECT
-    protoPayload.resource AS requestPath
-  FROM
-    `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
-  WHERE
-    _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')