Migrate the billing pipeline to flex template (#1100)

This is similar to the migration of the spec11 pipeline in #1073. Also removed
a few Dagger providers that are no longer needed.

TESTED=tested the dataflow job on alpha.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1100)
<!-- Reviewable:end -->

core/src/main/resources/google/registry/beam/init_sql_pipeline_metadata.json

@@ -5,7 +5,7 @@
     {
       "name": "registryEnvironment",
       "label": "The Registry environment.",
-      "helpText": "The Registry environment, required if environment-specific initialization is needed on worker VMs.",
+      "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.",
       "is_optional": true,
       "regexes": [
         "^[0-9A-Z_]+$"

core/src/main/resources/google/registry/beam/invoicing/sql/billing_events.sql

@@ -0,0 +1,101 @@
+#standardSQL
+  -- Copyright 2017 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- This query gathers all non-canceled billing events for a given
+  -- YEAR_MONTH in yyyy-MM format.
+
+SELECT
+  __key__.id AS id,
+  billingTime,
+  eventTime,
+  BillingEvent.clientId AS registrarId,
+  RegistrarData.accountId AS billingId,
+  RegistrarData.poNumber AS poNumber,
+  tld,
+  reason as action,
+  targetId as domain,
+  BillingEvent.domainRepoId as repositoryId,
+  periodYears as years,
+  BillingEvent.currency AS currency,
+  BillingEvent.amount as amount,
+  -- We'll strip out non-useful flags downstream
+  ARRAY_TO_STRING(flags, " ") AS flags
+FROM (
+  SELECT
+    *,
+    -- We store cost as "CURRENCY AMOUNT" such as "JPY 800" or "USD 20.00"
+    SPLIT(cost, ' ')[OFFSET(0)] AS currency,
+    SPLIT(cost, ' ')[OFFSET(1)] AS amount,
+    -- Extract everything after the first dot in the domain as the TLD
+    REGEXP_EXTRACT(targetId, r'[.](.+)') AS tld,
+    -- __key__.path looks like '"DomainBase", "<repoId>", ...'
+    REGEXP_REPLACE(SPLIT(__key__.path, ', ')[OFFSET(1)], '"', '')
+        AS domainRepoId,
+    COALESCE(cancellationMatchingBillingEvent.path,
+        __key__.path) AS cancellationMatchingPath
+  FROM
+    `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%ONETIME_TABLE%`
+    -- Only include real TLDs (filter prober data)
+  WHERE
+    REGEXP_EXTRACT(targetId, r'[.](.+)') IN (
+    SELECT
+      tldStr
+    FROM
+      `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
+    WHERE
+      invoicingEnabled IS TRUE) ) AS BillingEvent
+  -- Gather billing ID from registrar table
+  -- This is a 'JOIN' as opposed to 'LEFT JOIN' to filter out
+  -- non-billable registrars
+JOIN (
+  SELECT
+    __key__.name AS clientId,
+    billingIdentifier,
+    IFNULL(poNumber, '') AS poNumber,
+    r.billingAccountMap.currency[SAFE_OFFSET(index)] AS currency,
+    r.billingAccountMap.accountId[SAFE_OFFSET(index)] AS accountId
+  FROM
+    `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRAR_TABLE%` AS r,
+    UNNEST(GENERATE_ARRAY(0, ARRAY_LENGTH(r.billingAccountMap.currency) - 1))
+        AS index
+  WHERE billingAccountMap IS NOT NULL
+  AND type = 'REAL') AS RegistrarData
+ON
+  BillingEvent.clientId = RegistrarData.clientId
+  AND BillingEvent.currency = RegistrarData.currency
+  -- Gather cancellations
+LEFT JOIN (
+  SELECT __key__.id AS cancellationId,
+  COALESCE(refOneTime.path, refRecurring.path) AS cancelledEventPath,
+  eventTime as cancellationTime,
+  billingTime as cancellationBillingTime
+  FROM
+  (SELECT
+      *,
+      -- Count everything after first dot as TLD (to support multi-part TLDs).
+      REGEXP_EXTRACT(targetId, r'[.](.+)') AS tld
+    FROM
+      `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%CANCELLATION_TABLE%`)
+) AS Cancellation
+ON BillingEvent.cancellationMatchingPath = Cancellation.cancelledEventPath
+AND BillingEvent.billingTime = Cancellation.cancellationBillingTime
+WHERE billingTime BETWEEN TIMESTAMP('%FIRST_TIMESTAMP_OF_MONTH%')
+  AND TIMESTAMP('%LAST_TIMESTAMP_OF_MONTH%')
+-- Filter out canceled events
+AND Cancellation.cancellationId IS NULL
+ORDER BY
+  billingTime DESC,
+  id,
+  tld

core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json

@@ -0,0 +1,66 @@
+{
+  "name": "Invoice and Detailed Reports Generation",
+  "description": "An Apache Beam batch pipeline that reads from a Datastore export and generate monthly invoice and detailed reports, saving them on GCS.",
+  "parameters": [
+    {
+      "name": "registryEnvironment",
+      "label": "The Registry environment.",
+      "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.",
+      "is_optional": true,
+      "regexes": [
+        "^[0-9A-Z_]+$"
+      ]
+    },
+    {
+      "name": "isolationOverride",
+      "label": "The desired SQL transaction isolation level.",
+      "helpText": "The desired SQL transaction isolation level.",
+      "is_optional": true,
+      "regexes": [
+        "^[0-9A-Z_]+$"
+      ]
+    },
+    {
+      "name": "sqlWriteBatchSize",
+      "label": "SQL write batch size.",
+      "helpText": "The number of entities to write to the SQL database in one operation.",
+      "is_optional": true,
+      "regexes": [
+        "^[1-9][0-9]*$"
+      ]
+    },
+    {
+      "name": "sqlWriteShards",
+      "label": "Number of output shards to create when writing to SQL.",
+      "helpText": "Number of shards to create out of the data before writing to the SQL database. Please refer to the Javadoc of RegistryJpaIO.Write.shards() for how to choose this value.",
+      "is_optional": true,
+      "regexes": [
+        "^[1-9][0-9]*$"
+      ]
+    },
+    {
+      "name": "yearMonth",
+      "label": "The year and month we generate invoice and detailed reports for.",
+      "helpText": "The year and month for which the invoice and detailed reports are generated, in yyyy-MM format.",
+      "regexes": [
+        "^2[0-9]{3}-(0[1-9]|1[0-2])$"
+      ]
+    },
+    {
+      "name": "invoiceFilePrefix",
+      "label": "Filename prefix for the invoice CSV file.",
+      "helpText": "The prefix that will be applied to the invoice file.",
+      "regexes": [
+        "^[a-zA-Z0-9_\\-]+$"
+      ]
+    },
+    {
+      "name": "billingBucketUrl",
+      "label": "invoice and detailed reports upload dir.",
+      "helpText": "The root directory of the reports to upload.",
+      "regexes": [
+        "^gs:\\/\\/[^\\n\\r]+$"
+      ]
+    }
+  ]
+}

core/src/main/resources/google/registry/beam/spec11_pipeline_metadata.json

@@ -5,7 +5,7 @@
     {
       "name": "registryEnvironment",
       "label": "The Registry environment.",
-      "helpText": "The Registry environment, required if environment-specific initialization is needed on worker VMs.",
+      "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.",
       "is_optional": true,
       "regexes": [
         "^[0-9A-Z_]+$"
@@ -41,7 +41,7 @@
     {
       "name": "date",
       "label": "The date when the pipeline runs",
-      "helpText": "The date then the threat scan is performed, in yyyy-MM-dd format.",
+      "helpText": "The date when the threat scan is performed, in yyyy-MM-dd format.",
       "regexes": [
         "^2[0-9]{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$"
       ]