Read GKE logs in ICANN reports (#2673)

GKE logs are routed to a different dataset and the table is different.
The structs to look for are also different (jsonPayload vs textPayload
or protoPayload).

TESTED=Ran the resulting query in crash.

core/src/test/resources/google/registry/reporting/icann/attempted_adds_test.sql

@@ -19,7 +19,7 @@
   -- monthly App Engine logs, searching for all create commands and associating
   -- them with their corresponding registrars.
 
-  -- Example log generated by FlowReporter in App Engine logs:
+  -- Example log generated by FlowReporter in App Engine and GKE logs:
   --google.registry.flows.FlowReporter
   -- recordToLogs: FLOW-LOG-SIGNATURE-METADATA:
   --{"serverTrid":"oNwL2J2eRya7bh7c9oHIzg==-2360a","clientId":"ipmirror"
@@ -38,30 +38,42 @@ FROM (
     JSON_EXTRACT_SCALAR(json, '$.tld') AS tld,
     JSON_EXTRACT_SCALAR(json, '$.clientId') AS clientId,
     COUNT(json) AS count
-  FROM (
+  FROM ((
       -- Extract JSON metadata package from monthly logs
     SELECT
-      REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
-          AS json
-    FROM (
+      REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json
+    FROM
+      `domain-registry-alpha.appengine_logs._var_log_app_*`
+    WHERE
+      _TABLE_SUFFIX BETWEEN '20170901'
+      AND '20170930'
+      AND STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
+      -- Look for domain creates
+      AND REGEXP_CONTAINS(textPayload, r'"commandType":"create","resourceType":"domain"')
+      -- Filter prober data
+      AND NOT REGEXP_CONTAINS(textPayload, r'"prober-[a-z]{2}-((any)|(canary))"'))
+    UNION ALL (
+        -- Extract JSON metadata package from monthly logs
       SELECT
-        textPayload
+        REGEXP_EXTRACT(jsonPayload.message, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$') AS json
       FROM
-        `domain-registry-alpha.appengine_logs._var_log_app_*`
-      WHERE _TABLE_SUFFIX
-      BETWEEN '20170901'
-      AND '20170930')
-    WHERE STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
-    -- Look for domain creates
-    AND REGEXP_CONTAINS(
-       textPayload, r'"commandType":"create","resourceType":"domain"')
-    -- Filter prober data
-    AND NOT REGEXP_CONTAINS(
-        textPayload, r'"prober-[a-z]{2}-((any)|(canary))"') )
-  GROUP BY tld, clientId ) AS logs_table
+        `domain-registry-alpha.gke_logs.stderr_*`
+      WHERE
+        _TABLE_SUFFIX BETWEEN '20170901'
+        AND '20170930'
+        AND STARTS_WITH(jsonPayload.message, "FLOW-LOG-SIGNATURE-METADATA")
+        -- Look for domain creates
+        AND REGEXP_CONTAINS(jsonPayload.message, r'"commandType":"create","resourceType":"domain"')
+        -- Filter prober data
+        AND NOT REGEXP_CONTAINS(jsonPayload.message, r'"prober-[a-z]{2}-((any)|(canary))"')))
+  GROUP BY
+    tld,
+    clientId ) AS logs_table
 JOIN
   EXTERNAL_QUERY("projects/domain-registry-alpha/locations/us/connections/domain-registry-alpha-sql",
-              '''SELECT registrar_id, registrar_name FROM "Registrar";''') AS registrar_table
-  ON
-      logs_table.clientId = registrar_table.registrar_id
-  ORDER BY tld, registrar_name
+    '''SELECT registrar_id, registrar_name FROM "Registrar";''') AS registrar_table
+ON
+  logs_table.clientId = registrar_table.registrar_id
+ORDER BY
+  tld,
+  registrar_name

core/src/test/resources/google/registry/reporting/icann/epp_metrics_test.sql

@@ -35,14 +35,30 @@ FROM (
     JSON_EXTRACT_SCALAR(json,
       '$.icannActivityReportField') AS activityReportField
   FROM (
+    -- For reasons that I don't understand, if I directly select the three columns
+    -- from the union, BigQuery complains about column number mismatch, so I have to
+    -- make a temporary union table and select on it.
     SELECT
-      -- Extract the logged JSON payload.
-      REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
-      AS json
-    FROM `domain-registry-alpha.appengine_logs._var_log_app_*`
-    WHERE
-      STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
-      AND _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')) AS regexes
+      *
+    FROM (
+      SELECT
+        -- Extract the logged JSON payload.
+        REGEXP_EXTRACT(textPayload, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+        AS json
+      FROM `domain-registry-alpha.appengine_logs._var_log_app_*`
+      WHERE
+        STARTS_WITH(textPayload, "FLOW-LOG-SIGNATURE-METADATA")
+        AND _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')
+    UNION ALL (
+      SELECT
+        -- Extract the logged JSON payload.
+        REGEXP_EXTRACT(jsonPayload.message, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+        AS json
+      FROM `domain-registry-alpha.gke_logs.stderr_*`
+      WHERE
+        STARTS_WITH(jsonPayload.message, "FLOW-LOG-SIGNATURE-METADATA")
+        AND _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')
+  )) AS regexes
 JOIN
   -- Unnest the JSON-parsed tlds.
   UNNEST(regexes.tlds) AS tld

core/src/test/resources/google/registry/reporting/icann/monthly_logs_test.sql

@@ -13,13 +13,23 @@
   -- See the License for the specific language governing permissions and
   -- limitations under the License.
 
-  -- Query to fetch AppEngine request logs for the report month.
+  -- Query to fetch AppEngine and GKE request logs for the report month.
 
   -- START_OF_MONTH and END_OF_MONTH should be in YYYYMM01 format.
 
 SELECT
-  protoPayload.resource AS requestPath,
-FROM
-  `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
-WHERE
-  _TABLE_SUFFIX BETWEEN '20170901' AND '20170930'
+  *
+FROM (
+  SELECT
+    jsonPayload.httrequest.requesturl AS requestPath
+  FROM
+    `domain-registry-alpha.gke_logs.stderr_*`
+  WHERE
+    _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')
+UNION ALL (
+  SELECT
+    protoPayload.resource AS requestPath
+  FROM
+    `domain-registry-alpha.appengine_logs.appengine_googleapis_com_request_log_*`
+  WHERE
+    _TABLE_SUFFIX BETWEEN '20170901' AND '20170930')