Adds cloud scheduler and tasks deployer (#1999)

This is the last dependency on GAE pull queue, therefore we can delete
the pull queue config from queue.xml as well.

.gcloudignore

@@ -6,5 +6,4 @@ node_modules/
 repos/**
 **/.idea/
 *.jar
-!third_party/**/*.jar
 !/gradle/wrapper/**/*.jar

.github/workflows/codeql.yml

@@ -0,0 +1,63 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ 'master' ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ 'master' ]
+  schedule:
+    - cron: '24 4 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java', 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        queries: security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

.gitignore

@@ -14,7 +14,6 @@ gjf.out
 *.jar
 *.war
 *.ear
-!/third_party/**/*.jar
 
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*

.lgtm.yml

@@ -1,6 +0,0 @@
-extraction:
-  java:
-    prepare:
-      packages: "npm"
-    index:
-      java_version: "11"

README.md

@@ -1,8 +1,8 @@
 # Nomulus
 
-| Internal Build | FOSS Build | LGTM | License | Code Search |
-|:--------------:|:----------:|:----:|:-------:|:-----------:|
-|[![Build Status for Google Registry internal build](https://storage.googleapis.com/domain-registry-kokoro/internal/build.svg)](https://storage.googleapis.com/domain-registry-kokoro/internal/index.html)|[![Build Status for the open source build](https://storage.googleapis.com/domain-registry-kokoro/foss/build.svg)](https://storage.googleapis.com/domain-registry-kokoro/foss/index.html)|[![Total alerts](https://img.shields.io/lgtm/alerts/g/google/nomulus.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/google/nomulus/alerts/)|[![License for this repo](https://img.shields.io/github/license/google/nomulus.svg)](https://github.com/google/nomulus/blob/master/LICENSE)|[![Link to Code Search](https://www.gstatic.com/devopsconsole/images/oss/favicons/oss-32x32.png)](https://cs.opensource.google/nomulus/nomulus)|
+| Internal Build | FOSS Build | License | Code Search |
+|:--------------:|:----------:|:-------:|:-----------:|
+|[![Build Status for Google Registry internal build](https://storage.googleapis.com/domain-registry-kokoro/internal/build.svg)](https://storage.googleapis.com/domain-registry-kokoro/internal/index.html)|[![Build Status for the open source build](https://storage.googleapis.com/domain-registry-kokoro/foss/build.svg)](https://storage.googleapis.com/domain-registry-kokoro/foss/index.html)|[![License for this repo](https://img.shields.io/github/license/google/nomulus.svg)](https://github.com/google/nomulus/blob/master/LICENSE)|[![Link to Code Search](https://www.gstatic.com/devopsconsole/images/oss/favicons/oss-32x32.png)](https://cs.opensource.google/nomulus/nomulus)|
 
 ![Nomulus logo](./nomulus-logo.png)
 

appengine_war.gradle

@@ -47,6 +47,10 @@ war {
 
 if (project.path == ":services:default") {
     war {
+        from("${rootDir}/console-webapp/dist/console-webapp") {
+            include "**/*"
+            into("console")
+        }
         from("${coreResourcesDir}/google/registry/ui") {
             include "registrar_bin.js"
             if (environment != "production") {
@@ -99,8 +103,10 @@ explodeWar.doLast {
     file("${it.explodedAppDirectory}/WEB-INF/lib/tools.jar").setWritable(true)
 }
 
+appengineDeployAll.finalizedBy ':deployCloudSchedulerAndQueue'
 rootProject.deploy.dependsOn appengineDeployAll
 rootProject.stage.dependsOn appengineStage
+tasks['war'].dependsOn ':console-webapp:buildConsoleWebappProd'
 tasks['war'].dependsOn ':core:compileProdJS'
 tasks['war'].dependsOn ':core:processResources'
 tasks['war'].dependsOn ':core:jar'

build.gradle

@@ -355,8 +355,6 @@ subprojects {
     }
   }
 
-  if (project.name == 'third_party') return
-
   project.tasks.test.dependsOn runPresubmits
 
   def commonlyExcludedResources = ['**/*.java', '**/BUILD']
@@ -528,6 +526,10 @@ task javaIncrementalFormatApply {
 
 task javadoc(type: Javadoc) {
   source javadocSource
+  // Java 11.0.17 has the following bug that affects annotation handling on
+  // package-info.java:
+  // https://bugs.openjdk.org/browse/JDK-8222091
+  exclude "**/package-info.java"
   classpath = files(javadocClasspath)
   destinationDir = file("${buildDir}/docs/javadoc")
   options.encoding = "UTF-8"
@@ -549,12 +551,34 @@ task coreDev {
   dependsOn 'javadoc'
   dependsOn 'checkDependenciesDotGradle'
   dependsOn 'checkLicense'
+  dependsOn ':console-webapp:runConsoleWebappUnitTests'
   dependsOn ':core:check'
   dependsOn 'assemble'
 }
 
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
 
+// Runs the script, which deploys cloud scheduler and tasks based on the config
+task deployCloudSchedulerAndQueue {
+  doLast {
+    def env = environment
+    if (!prodOrSandboxEnv) {
+      exec {
+        commandLine 'go', 'run',
+                "${rootDir}/release/builder/deployCloudSchedulerAndQueue.go",
+                "${rootDir}/core/src/main/java/google/registry/env/${env}/default/WEB-INF/cloud-scheduler-tasks.xml",
+                "domain-registry-${env}"
+      }
+      exec {
+        commandLine 'go', 'run',
+                "${rootDir}/release/builder/deployCloudSchedulerAndQueue.go",
+                "${rootDir}/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml",
+                "domain-registry-${env}"
+      }
+    }
+  }
+}
+
 // disable javadoc in subprojects, these will break because they don't have
 // the correct classpath (see above).
 gradle.taskGraph.whenReady { graph ->

buildSrc/gradle.lockfile

@@ -4,38 +4,37 @@
 antlr:antlr:2.7.7=checkstyle
 aopalliance:aopalliance:1.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 args4j:args4j:2.0.23=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson.core:jackson-core:2.13.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson:jackson-bom:2.13.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-core:2.14.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson:jackson-bom:2.14.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.ben-manes.caffeine:caffeine:2.7.0=annotationProcessor,testAnnotationProcessor
 com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,testAnnotationProcessor
 com.google.android:annotations:4.1.1.4=testRuntimeClasspath
-com.google.api-client:google-api-client:2.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:gapic-google-cloud-storage-v2:2.14.0-alpha=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-storage-v2:2.14.0-alpha=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-iam-v1:1.6.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-storage-v2:2.14.0-alpha=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-common-protos:2.9.6=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-iam-v1:1.6.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api:api-common:2.2.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api:gax-grpc:2.19.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api:gax-httpjson:0.104.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api:gax:2.19.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api-client:google-api-client:2.1.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:gapic-google-cloud-storage-v2:2.17.2-alpha=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-storage-v2:2.17.2-alpha=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-storage-v2:2.17.2-alpha=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-common-protos:2.13.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-iam-v1:1.8.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api:api-common:2.5.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api:gax-grpc:2.22.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api:gax-httpjson:0.107.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api:gax:2.22.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.auth:google-auth-library-credentials:1.12.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.auth:google-auth-library-oauth2-http:1.12.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.auto.value:auto-value-annotations:1.10=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.auto.value:auto-value:1.10=annotationProcessor
+com.google.auth:google-auth-library-credentials:1.14.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.auth:google-auth-library-oauth2-http:1.14.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.auto.value:auto-value-annotations:1.10.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.auto.value:auto-value:1.10.1=annotationProcessor,compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.auto:auto-common:0.10=annotationProcessor,testAnnotationProcessor
-com.google.cloud:google-cloud-core-grpc:2.8.22=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-core-http:2.8.22=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-core:2.8.22=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-storage:2.14.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-core-grpc:2.9.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-core-http:2.9.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-core:2.9.4=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-storage:2.17.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.code.findbugs:jFormatString:3.0.0=annotationProcessor,testAnnotationProcessor
 com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,checkstyle,compileClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
-com.google.code.gson:gson:2.9.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.code.gson:gson:2.10.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.common.html.types:types:1.0.6=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.errorprone:error_prone_annotation:2.3.4=annotationProcessor,testAnnotationProcessor
-com.google.errorprone:error_prone_annotations:2.11.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.errorprone:error_prone_annotations:2.18.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.errorprone:error_prone_annotations:2.3.4=annotationProcessor,checkstyle,testAnnotationProcessor
 com.google.errorprone:error_prone_check_api:2.3.4=annotationProcessor,testAnnotationProcessor
 com.google.errorprone:error_prone_core:2.3.4=annotationProcessor,testAnnotationProcessor
@@ -46,19 +45,19 @@ com.google.guava:guava:27.0.1-jre=annotationProcessor,testAnnotationProcessor
 com.google.guava:guava:29.0-jre=checkstyle
 com.google.guava:guava:31.1-jre=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,compileClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-apache-v2:1.42.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-appengine:1.42.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-gson:1.42.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-jackson2:1.42.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client:1.42.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-apache-v2:1.42.3=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-appengine:1.42.3=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-gson:1.42.3=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-jackson2:1.42.3=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client:1.42.3=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.inject.extensions:guice-multibindings:4.1.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.inject:guice:4.1.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.j2objc:j2objc-annotations:1.1=annotationProcessor,testAnnotationProcessor
 com.google.j2objc:j2objc-annotations:1.3=checkstyle,compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.jsinterop:jsinterop-annotations:1.0.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.oauth-client:google-oauth-client:1.34.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.protobuf:protobuf-java-util:3.21.8=compileClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.protobuf:protobuf-java:3.21.8=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.protobuf:protobuf-java-util:3.21.12=compileClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.protobuf:protobuf-java:3.21.12=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.protobuf:protobuf-java:3.4.0=annotationProcessor,testAnnotationProcessor
 com.google.re2j:re2j:1.6=testRuntimeClasspath
 com.google.template:soy:2021-02-01=compileClasspath,testCompileClasspath,testRuntimeClasspath
@@ -68,33 +67,33 @@ com.googlecode.java-diff-utils:diffutils:1.3.0=annotationProcessor,testAnnotatio
 com.ibm.icu:icu4j:57.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
 com.puppycrawl.tools:checkstyle:8.37=checkstyle
 commons-beanutils:commons-beanutils:1.9.4=checkstyle
-commons-codec:commons-codec:1.11=compileClasspath,testCompileClasspath,testRuntimeClasspath
+commons-codec:commons-codec:1.15=compileClasspath,testCompileClasspath,testRuntimeClasspath
 commons-collections:commons-collections:3.2.2=checkstyle
 commons-logging:commons-logging:1.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
 info.picocli:picocli:4.5.2=checkstyle
-io.grpc:grpc-alts:1.50.1=testRuntimeClasspath
-io.grpc:grpc-api:1.50.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-auth:1.50.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-context:1.50.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-core:1.50.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-googleapis:1.50.1=testRuntimeClasspath
-io.grpc:grpc-grpclb:1.50.1=testRuntimeClasspath
-io.grpc:grpc-netty-shaded:1.50.1=testRuntimeClasspath
-io.grpc:grpc-protobuf-lite:1.50.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-protobuf:1.50.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-services:1.50.1=testRuntimeClasspath
-io.grpc:grpc-stub:1.50.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-xds:1.50.1=testRuntimeClasspath
+io.grpc:grpc-alts:1.52.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-api:1.52.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-auth:1.52.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-context:1.52.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-core:1.52.1=testRuntimeClasspath
+io.grpc:grpc-googleapis:1.52.1=testRuntimeClasspath
+io.grpc:grpc-grpclb:1.52.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-netty-shaded:1.52.1=testRuntimeClasspath
+io.grpc:grpc-protobuf-lite:1.52.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-protobuf:1.52.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-services:1.52.1=testRuntimeClasspath
+io.grpc:grpc-stub:1.52.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-xds:1.52.1=testRuntimeClasspath
 io.opencensus:opencensus-api:0.31.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
 io.opencensus:opencensus-contrib-http-util:0.31.1=compileClasspath,testCompileClasspath,testRuntimeClasspath
 io.opencensus:opencensus-proto:0.2.0=testRuntimeClasspath
-io.perfmark:perfmark-api:0.25.0=testRuntimeClasspath
+io.perfmark:perfmark-api:0.26.0=testRuntimeClasspath
 javax.annotation:javax.annotation-api:1.3.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
 javax.annotation:jsr250-api:1.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 javax.inject:javax.inject:1=compileClasspath,testCompileClasspath,testRuntimeClasspath
 junit:junit:4.13.2=testCompileClasspath,testRuntimeClasspath
-net.bytebuddy:byte-buddy-agent:1.12.16=testCompileClasspath,testRuntimeClasspath
-net.bytebuddy:byte-buddy:1.12.16=testCompileClasspath,testRuntimeClasspath
+net.bytebuddy:byte-buddy-agent:1.12.22=testCompileClasspath,testRuntimeClasspath
+net.bytebuddy:byte-buddy:1.12.22=testCompileClasspath,testRuntimeClasspath
 net.sf.saxon:Saxon-HE:10.3=checkstyle
 org.antlr:antlr4-runtime:4.8-1=checkstyle
 org.apache.commons:commons-lang3:3.12.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
@@ -104,41 +103,40 @@ org.apache.httpcomponents:httpcore:4.4.15=compileClasspath,testCompileClasspath,
 org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
 org.checkerframework:checker-qual:2.11.1=checkstyle
 org.checkerframework:checker-qual:3.0.0=annotationProcessor,testAnnotationProcessor
-org.checkerframework:checker-qual:3.26.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
+org.checkerframework:checker-qual:3.29.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.checkerframework:dataflow:3.0.0=annotationProcessor,testAnnotationProcessor
 org.checkerframework:javacutil:3.0.0=annotationProcessor,testAnnotationProcessor
 org.codehaus.mojo:animal-sniffer-annotations:1.17=annotationProcessor,testAnnotationProcessor
 org.codehaus.mojo:animal-sniffer-annotations:1.22=testRuntimeClasspath
-org.conscrypt:conscrypt-openjdk-uber:2.5.2=testRuntimeClasspath
+org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.hamcrest:hamcrest-core:1.3=testCompileClasspath,testRuntimeClasspath
-org.jacoco:org.jacoco.agent:0.8.6=jacocoAgent,jacocoAnt
-org.jacoco:org.jacoco.ant:0.8.6=jacocoAnt
-org.jacoco:org.jacoco.core:0.8.6=jacocoAnt
-org.jacoco:org.jacoco.report:0.8.6=jacocoAnt
+org.jacoco:org.jacoco.agent:0.8.7=jacocoAgent,jacocoAnt
+org.jacoco:org.jacoco.ant:0.8.7=jacocoAnt
+org.jacoco:org.jacoco.core:0.8.7=jacocoAnt
+org.jacoco:org.jacoco.report:0.8.7=jacocoAnt
 org.javassist:javassist:3.26.0-GA=checkstyle
 org.json:json:20160212=compileClasspath,testCompileClasspath,testRuntimeClasspath
-org.junit.jupiter:junit-jupiter-api:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.jupiter:junit-jupiter-engine:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-commons:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-engine:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit:junit-bom:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.mockito:mockito-core:4.8.1=testCompileClasspath,testRuntimeClasspath
-org.objenesis:objenesis:3.2=testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-api:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-engine:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-commons:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-engine:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit:junit-bom:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.mockito:mockito-core:5.0.0=testCompileClasspath,testRuntimeClasspath
+org.objenesis:objenesis:3.3=testRuntimeClasspath
 org.opentest4j:opentest4j:1.2.0=testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm-analysis:7.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
-org.ow2.asm:asm-analysis:8.0.1=jacocoAnt
+org.ow2.asm:asm-analysis:9.1=jacocoAnt
 org.ow2.asm:asm-commons:7.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
-org.ow2.asm:asm-commons:8.0.1=jacocoAnt
+org.ow2.asm:asm-commons:9.1=jacocoAnt
 org.ow2.asm:asm-tree:7.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
-org.ow2.asm:asm-tree:8.0.1=jacocoAnt
+org.ow2.asm:asm-tree:9.1=jacocoAnt
 org.ow2.asm:asm-util:7.0=compileClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:7.0=compileClasspath
-org.ow2.asm:asm:8.0.1=jacocoAnt
-org.ow2.asm:asm:9.1=testCompileClasspath,testRuntimeClasspath
+org.ow2.asm:asm:9.1=jacocoAnt,testCompileClasspath,testRuntimeClasspath
 org.pcollections:pcollections:2.1.2=annotationProcessor,testAnnotationProcessor
 org.plumelib:plume-util:1.0.6=annotationProcessor,testAnnotationProcessor
 org.plumelib:reflection-util:0.0.2=annotationProcessor,testAnnotationProcessor
 org.plumelib:require-javadoc:0.1.0=annotationProcessor,testAnnotationProcessor
 org.reflections:reflections:0.9.12=checkstyle
-org.threeten:threetenbp:1.6.3=compileClasspath,testCompileClasspath,testRuntimeClasspath
+org.threeten:threetenbp:1.6.5=compileClasspath,testCompileClasspath,testRuntimeClasspath
 empty=

buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java

@@ -17,7 +17,6 @@ package google.registry.gradle.plugin;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import google.registry.gradle.plugin.ProjectData.TaskData;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Supplier;

common/build.gradle

@@ -38,6 +38,7 @@ configurations {
 
   // All testing util classes. Other projects may declare dependency as:
   // testImplementation project(path: 'common', configuration: 'testing')
+  create("testing")
   testing.extendsFrom testingCompileOnly
 }
 

common/gradle.lockfile

@@ -32,7 +32,7 @@ commons-collections:commons-collections:3.2.2=checkstyle
 info.picocli:picocli:4.5.2=checkstyle
 io.github.java-diff-utils:java-diff-utils:4.12=compileClasspath,default,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath,testing,testingCompileClasspath
 javax.inject:javax.inject:1=compileClasspath,default,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath,testing,testingCompileClasspath
-joda-time:joda-time:2.12.1=compileClasspath,default,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath,testing,testingCompileClasspath
+joda-time:joda-time:2.12.2=compileClasspath,default,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath,testing,testingCompileClasspath
 junit:junit:4.13.2=default,testCompileClasspath,testRuntimeClasspath,testing,testingCompileClasspath
 net.sf.saxon:Saxon-HE:10.3=checkstyle
 org.antlr:antlr4-runtime:4.8-1=checkstyle
@@ -45,22 +45,21 @@ org.checkerframework:dataflow:3.0.0=annotationProcessor,errorprone,testAnnotatio
 org.checkerframework:javacutil:3.0.0=annotationProcessor,errorprone,testAnnotationProcessor,testingAnnotationProcessor
 org.codehaus.mojo:animal-sniffer-annotations:1.17=annotationProcessor,errorprone,testAnnotationProcessor,testingAnnotationProcessor
 org.hamcrest:hamcrest-core:1.3=default,testCompileClasspath,testRuntimeClasspath,testing,testingCompileClasspath
-org.jacoco:org.jacoco.agent:0.8.6=jacocoAgent,jacocoAnt
-org.jacoco:org.jacoco.ant:0.8.6=jacocoAnt
-org.jacoco:org.jacoco.core:0.8.6=jacocoAnt
-org.jacoco:org.jacoco.report:0.8.6=jacocoAnt
+org.jacoco:org.jacoco.agent:0.8.7=jacocoAgent,jacocoAnt
+org.jacoco:org.jacoco.ant:0.8.7=jacocoAnt
+org.jacoco:org.jacoco.core:0.8.7=jacocoAnt
+org.jacoco:org.jacoco.report:0.8.7=jacocoAnt
 org.javassist:javassist:3.26.0-GA=checkstyle
-org.junit.jupiter:junit-jupiter-api:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.jupiter:junit-jupiter-engine:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-commons:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-engine:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit:junit-bom:5.9.1=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-api:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-engine:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-commons:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-engine:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit:junit-bom:5.9.2=testCompileClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.2.0=testCompileClasspath,testRuntimeClasspath
-org.ow2.asm:asm-analysis:8.0.1=jacocoAnt
-org.ow2.asm:asm-commons:8.0.1=jacocoAnt
-org.ow2.asm:asm-tree:8.0.1=jacocoAnt
-org.ow2.asm:asm:8.0.1=jacocoAnt
-org.ow2.asm:asm:9.1=compileClasspath,default,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath,testing,testingCompileClasspath
+org.ow2.asm:asm-analysis:9.1=jacocoAnt
+org.ow2.asm:asm-commons:9.1=jacocoAnt
+org.ow2.asm:asm-tree:9.1=jacocoAnt
+org.ow2.asm:asm:9.1=compileClasspath,default,deploy_jar,jacocoAnt,runtimeClasspath,testCompileClasspath,testRuntimeClasspath,testing,testingCompileClasspath
 org.pcollections:pcollections:2.1.2=annotationProcessor,errorprone,testAnnotationProcessor,testingAnnotationProcessor
 org.plumelib:plume-util:1.0.6=annotationProcessor,errorprone,testAnnotationProcessor,testingAnnotationProcessor
 org.plumelib:reflection-util:0.0.2=annotationProcessor,errorprone,testAnnotationProcessor,testingAnnotationProcessor

common/src/main/java/google/registry/util/DateTimeUtils.java

@@ -35,7 +35,7 @@ public abstract class DateTimeUtils {
    *
    * <p>This value is (2^63-1)/1000 rounded down. AppEngine stores dates as 64 bit microseconds, but
    * Java uses milliseconds, so this is the largest representable date that will survive a
-   * round-trip through Datastore.
+   * round-trip through the database.
    */
   public static final DateTime END_OF_TIME = new DateTime(Long.MAX_VALUE / 1000, DateTimeZone.UTC);
 

config/checkstyle/checkstyle.xml

@@ -43,12 +43,6 @@ by Joshua Bloch in his book Effective Java -->
     <property name="message" value='Your Javadocs appear to use invalid &lt;a&gt; tag syntax in @see tags. Please use the correct syntax: @see &lt;a href="http(s)://your_url"&gt;url_description&lt;/a&gt;'/>
   </module>
 
-  <!-- Checks that our Ofy wrapper is used instead of the "real" ofy(). -->
-  <module name="RegexpSingleline">
-    <property name="format" value="com\.googlecode\.objectify\.ObjectifyService\.ofy"/>
-    <property name="message" value="Use google.registry.model.ofy.ObjectifyService.ofy(). Do not use com.googlecode.objectify.v4.ObjectifyService.ofy()."/>
-  </module>
-
   <!-- Checks that java.util.Optional is used instead of Guava's Optional. -->
   <module name="RegexpSingleline">
     <property name="format" value="com\.google\.common\.base\.Optional"/>
@@ -58,7 +52,7 @@ by Joshua Bloch in his book Effective Java -->
   <!-- Checks that the deprecated ExpectedException is not used. -->
   <module name="RegexpSingleline">
     <property name="format" value="org\.junit\.rules\.ExpectedException"/>
-    <property name="message" value="Use assertThrows and expectThrows from JUnitBackports instead of the deprecated methods on ExpectedException."/>
+    <property name="message" value="Use assertThrows and expectThrows instead of the deprecated methods on ExpectedException."/>
   </module>
 
   <module name="LineLength">

config/checkstyle/suppressions.xml

@@ -9,6 +9,4 @@
   <suppress files="[/\\].*[/\\]generated.*[/\\]" checks="."/>
   <!-- Ignore Javadoc checks in test files -->
   <suppress files="[/\\].*[/\\]src/test/java/.*[/\\]" checks="JavadocType"/>
-  <!-- ofy() regex check doesn't apply to these files -->
-  <suppress files="AugmentedDeleter.java|AugmentedSaver.java|Ofy.java" checks="RegexpSingleline"/>
 </suppressions>

config/dependency-license/allowed_licenses.json

@@ -270,6 +270,10 @@
       "moduleLicense": "Public Domain",
       "moduleName": "org.tukaani:xz"
     },
+    {
+      "moduleLicense": "Public Domain",
+      "moduleName": "org.json:json"
+    },
     {
       // "Apache License, Version 2.0".
       "moduleLicense": null,

config/presubmits.py

@@ -25,7 +25,7 @@ import textwrap
 import re
 
 # We should never analyze any generated files
-UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/", ".gradle/"}
+UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/", ".gradle/", "/dist/", "karma.conf.js", "polyfills.ts", "test.ts"}
 # We can't rely on CI to have the Enum package installed so we do this instead.
 FORBIDDEN = 1
 REQUIRED = 2
@@ -86,7 +86,7 @@ PRESUBMITS = {
     # License check
     PresubmitCheck(
         r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
-        ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
+        ("java", "js", "soy", "sql", "py", "sh", "gradle", "ts"), {
             ".git", "/build/", "/generated/", "/generated_tests/",
             "node_modules/", "LoggerConfig.java", "registrar_bin.",
             "registrar_dbg.", "google-java-format-diff.py",
@@ -95,7 +95,7 @@ PRESUBMITS = {
         "File did not include the license header.",
 
     # Files must end in a newline
-    PresubmitCheck(r".*\n$", ("java", "js", "soy", "sql", "py", "sh", "gradle"),
+    PresubmitCheck(r".*\n$", ("java", "js", "soy", "sql", "py", "sh", "gradle", "ts"),
                    {"node_modules/"}, REQUIRED):
         "Source files must end in a newline.",
 
@@ -109,15 +109,6 @@ PRESUBMITS = {
         "System.(out|err).println is only allowed in tools/ packages. Please "
         "use a logger instead.",
 
-    # ObjectifyService.register is restricted to main/ or AppEngineExtension.
-    PresubmitCheck(
-        r".*\bObjectifyService\.register", "java", {
-            "/build/", "/generated/", "node_modules/", "src/main/",
-            "AppEngineExtension.java"
-        }):
-      "ObjectifyService.register(...) is not allowed in tests. Please use "
-      "AppEngineExtension.register(...) instead.",
-
     # PostgreSQLContainer instantiation must specify docker tag
     # TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
     PresubmitCheck(

console-webapp/.editorconfig

@@ -0,0 +1,16 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false

console-webapp/.gitignore

@@ -0,0 +1,42 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+
+# Compiled output
+/dist
+/tmp
+/out-tsc
+/bazel-out
+
+# Node
+/node_modules
+npm-debug.log
+yarn-error.log
+
+# IDEs and editors
+.idea/
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# Visual Studio Code
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history/*
+
+# Miscellaneous
+/.angular/cache
+.sass-cache/
+/connect.lock
+/coverage
+/libpeerconnection.log
+testem.log
+/typings
+
+# System files
+.DS_Store
+Thumbs.db

console-webapp/README.md

@@ -0,0 +1,52 @@
+# ConsoleWebapp
+
+A web application for managing [Nomulus](https://github.com/google/nomulus).
+
+## Status
+
+Console webapp is currently under active development and some parts of it are
+expected to change.
+
+## Deployment
+
+Webapp is deployed with the nomulus default service war to Google App Engine.
+During nomulus default service war build task, gradle script triggers the
+following:
+
+1) Console webapp build script `buildConsoleWebappProd`, which installs
+   dependencies, assembles a compiled ts -> js, minified, optimized static
+   artifact (html, css, js)
+2) Artifact assembled in step 1 then gets copied to core project web artifact
+   location, so that it can be deployed with the rest of the core webapp
+
+## Development server
+
+Run `npm run start:dev` to start both webapp dev server and API server instance.
+Navigate to `http://localhost:4200/`. The application will automatically reload
+if you change any of the source files.
+
+## Code scaffolding
+
+Run `ng generate component component-name` to generate a new component. You can
+also use `ng generate directive|pipe|service|class|guard|interface|enum|module`.
+
+## Build
+
+Run `ng build` to build the project. The build artifacts will be stored in
+the `dist/` directory.
+
+## Running unit tests
+
+Run `ng test` to execute the unit tests
+via [Karma](https://karma-runner.github.io).
+
+## Running end-to-end tests
+
+Run `ng e2e` to execute the end-to-end tests via a platform of your choice. To
+use this command, you need to first add a package that implements end-to-end
+testing capabilities.
+
+## Further help
+
+To get more help on the Angular CLI use `ng help` or go check out
+the [Angular CLI Overview and Command Reference](https://angular.io/cli) page.

console-webapp/angular.json

@@ -0,0 +1,109 @@
+{
+  "$schema": "./node_modules/@angular/cli/lib/config/schema.json",
+  "version": 1,
+  "newProjectRoot": "projects",
+  "projects": {
+    "console-webapp": {
+      "projectType": "application",
+      "schematics": {
+        "@schematics/angular:component": {
+          "style": "less"
+        }
+      },
+      "root": "",
+      "sourceRoot": "src",
+      "prefix": "app",
+      "architect": {
+        "build": {
+          "builder": "@angular-devkit/build-angular:browser",
+          "options": {
+            "outputPath": "dist/console-webapp",
+            "index": "src/index.html",
+            "main": "src/main.ts",
+            "polyfills": "src/polyfills.ts",
+            "tsConfig": "tsconfig.app.json",
+            "inlineStyleLanguage": "less",
+            "assets": [
+              "src/favicon.ico",
+              "src/assets"
+            ],
+            "styles": [
+              "./node_modules/@angular/material/prebuilt-themes/indigo-pink.css",
+              "src/styles.less"
+            ],
+            "scripts": []
+          },
+          "configurations": {
+            "production": {
+              "budgets": [
+                {
+                  "type": "initial",
+                  "maximumWarning": "500kb",
+                  "maximumError": "1mb"
+                },
+                {
+                  "type": "anyComponentStyle",
+                  "maximumWarning": "2kb",
+                  "maximumError": "4kb"
+                }
+              ],
+              "fileReplacements": [
+                {
+                  "replace": "src/environments/environment.ts",
+                  "with": "src/environments/environment.prod.ts"
+                }
+              ],
+              "outputHashing": "all"
+            },
+            "development": {
+              "buildOptimizer": false,
+              "optimization": false,
+              "vendorChunk": true,
+              "extractLicenses": false,
+              "sourceMap": true,
+              "namedChunks": true
+            }
+          },
+          "defaultConfiguration": "production"
+        },
+        "serve": {
+          "builder": "@angular-devkit/build-angular:dev-server",
+          "configurations": {
+            "production": {
+              "browserTarget": "console-webapp:build:production"
+            },
+            "development": {
+              "browserTarget": "console-webapp:build:development"
+            }
+          },
+          "defaultConfiguration": "development"
+        },
+        "extract-i18n": {
+          "builder": "@angular-devkit/build-angular:extract-i18n",
+          "options": {
+            "browserTarget": "console-webapp:build"
+          }
+        },
+        "test": {
+          "builder": "@angular-devkit/build-angular:karma",
+          "options": {
+            "main": "src/test.ts",
+            "polyfills": "src/polyfills.ts",
+            "tsConfig": "tsconfig.spec.json",
+            "karmaConfig": "karma.conf.js",
+            "inlineStyleLanguage": "less",
+            "assets": [
+              "src/favicon.ico",
+              "src/assets"
+            ],
+            "styles": [
+              "./node_modules/@angular/material/prebuilt-themes/indigo-pink.css",
+              "src/styles.less"
+            ],
+            "scripts": []
+          }
+        }
+      }
+    }
+  }
+}

console-webapp/build.gradle

@@ -0,0 +1,54 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+def consoleDir = "${rootDir}/console-webapp"
+
+clean {
+  delete "${consoleDir}/node_modules"
+  delete "${consoleDir}/dist"
+}
+
+task npmInstallDeps(type: Exec) {
+  workingDir "${consoleDir}/"
+  executable 'npm'
+  args 'i', '--no-audit', '--no-fund', '--loglevel=error'
+}
+
+task runConsoleWebappLocally(type: Exec) {
+  workingDir "${consoleDir}/"
+  executable 'npm'
+  args 'run', 'start:dev'
+}
+
+task runConsoleWebappUnitTests(type: Exec) {
+  workingDir "${consoleDir}/"
+  executable 'npm'
+  args 'run', 'test'
+}
+
+task buildConsoleWebappNonProd(type: Exec) {
+  workingDir "${consoleDir}/"
+  executable 'npm'
+  args 'run', 'build'
+}
+
+// Keeping the same as non prod for now before we figure out optimization we want to include
+task buildConsoleWebappProd(type: Exec) {
+  workingDir "${consoleDir}/"
+  executable 'npm'
+  args 'run', 'build'
+}
+
+tasks.runConsoleWebappUnitTests.dependsOn(tasks.npmInstallDeps)
+tasks.buildConsoleWebappProd.dependsOn(tasks.npmInstallDeps)

console-webapp/buildscript-gradle.lockfile

@@ -0,0 +1,4 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+empty=classpath

console-webapp/dev-proxy.config.json

@@ -0,0 +1,7 @@
+{
+  "/registrar":
+  {
+    "target": "http://localhost:8080/registrar",
+    "secure": false
+  }
+}

console-webapp/gradle.lockfile

@@ -0,0 +1,48 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+antlr:antlr:2.7.7=checkstyle
+com.github.ben-manes.caffeine:caffeine:2.7.0=annotationProcessor,errorprone,testAnnotationProcessor
+com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.auto:auto-common:0.10=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.code.findbugs:jFormatString:3.0.0=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,checkstyle,errorprone,testAnnotationProcessor
+com.google.errorprone:error_prone_annotation:2.3.4=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.errorprone:error_prone_annotations:2.3.4=annotationProcessor,checkstyle,errorprone,testAnnotationProcessor
+com.google.errorprone:error_prone_check_api:2.3.4=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.errorprone:error_prone_core:2.3.4=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.errorprone:error_prone_type_annotations:2.3.4=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.guava:failureaccess:1.0.1=annotationProcessor,checkstyle,errorprone,testAnnotationProcessor
+com.google.guava:guava:27.0.1-jre=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.guava:guava:29.0-jre=checkstyle
+com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,errorprone,testAnnotationProcessor
+com.google.j2objc:j2objc-annotations:1.1=annotationProcessor,errorprone,testAnnotationProcessor
+com.google.j2objc:j2objc-annotations:1.3=checkstyle
+com.google.protobuf:protobuf-java:3.4.0=annotationProcessor,errorprone,testAnnotationProcessor
+com.googlecode.java-diff-utils:diffutils:1.3.0=annotationProcessor,errorprone,testAnnotationProcessor
+com.puppycrawl.tools:checkstyle:8.37=checkstyle
+commons-beanutils:commons-beanutils:1.9.4=checkstyle
+commons-collections:commons-collections:3.2.2=checkstyle
+info.picocli:picocli:4.5.2=checkstyle
+net.sf.saxon:Saxon-HE:10.3=checkstyle
+org.antlr:antlr4-runtime:4.8-1=checkstyle
+org.checkerframework:checker-qual:2.11.1=checkstyle
+org.checkerframework:checker-qual:3.0.0=annotationProcessor,errorprone,testAnnotationProcessor
+org.checkerframework:dataflow:3.0.0=annotationProcessor,errorprone,testAnnotationProcessor
+org.checkerframework:javacutil:3.0.0=annotationProcessor,errorprone,testAnnotationProcessor
+org.codehaus.mojo:animal-sniffer-annotations:1.17=annotationProcessor,errorprone,testAnnotationProcessor
+org.jacoco:org.jacoco.agent:0.8.7=jacocoAgent,jacocoAnt
+org.jacoco:org.jacoco.ant:0.8.7=jacocoAnt
+org.jacoco:org.jacoco.core:0.8.7=jacocoAnt
+org.jacoco:org.jacoco.report:0.8.7=jacocoAnt
+org.javassist:javassist:3.26.0-GA=checkstyle
+org.ow2.asm:asm-analysis:9.1=jacocoAnt
+org.ow2.asm:asm-commons:9.1=jacocoAnt
+org.ow2.asm:asm-tree:9.1=jacocoAnt
+org.ow2.asm:asm:9.1=jacocoAnt
+org.pcollections:pcollections:2.1.2=annotationProcessor,errorprone,testAnnotationProcessor
+org.plumelib:plume-util:1.0.6=annotationProcessor,errorprone,testAnnotationProcessor
+org.plumelib:reflection-util:0.0.2=annotationProcessor,errorprone,testAnnotationProcessor
+org.plumelib:require-javadoc:0.1.0=annotationProcessor,errorprone,testAnnotationProcessor
+org.reflections:reflections:0.9.12=checkstyle
+empty=archives,compileClasspath,default,deploy_jar,errorproneJavac,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

console-webapp/karma.conf.js

@@ -0,0 +1,44 @@
+// Karma configuration file, see link for more information
+// https://karma-runner.github.io/1.0/config/configuration-file.html
+
+module.exports = function (config) {
+  config.set({
+    basePath: '',
+    frameworks: ['jasmine', '@angular-devkit/build-angular'],
+    plugins: [
+      require('karma-jasmine'),
+      require('karma-chrome-launcher'),
+      require('karma-jasmine-html-reporter'),
+      require('karma-coverage'),
+      require('@angular-devkit/build-angular/plugins/karma')
+    ],
+    client: {
+      jasmine: {
+        // you can add configuration options for Jasmine here
+        // the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html
+        // for example, you can disable the random execution with `random: false`
+        // or set a specific seed with `seed: 4321`
+      },
+      clearContext: false // leave Jasmine Spec Runner output visible in browser
+    },
+    jasmineHtmlReporter: {
+      suppressAll: true // removes the duplicated traces
+    },
+    coverageReporter: {
+      dir: require('path').join(__dirname, './coverage/console-webapp'),
+      subdir: '.',
+      reporters: [
+        { type: 'html' },
+        { type: 'text-summary' }
+      ]
+    },
+    reporters: ['progress', 'kjhtml'],
+    port: 9876,
+    colors: true,
+    logLevel: config.LOG_INFO,
+    autoWatch: true,
+    browsers: ['Chrome'],
+    singleRun: false,
+    restartOnFileChange: true
+  });
+};

console-webapp/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "console-webapp",
+  "version": "0.0.0",
+  "scripts": {
+    "ng": "ng",
+    "start": "ng serve",
+    "build": "ng build --base-href=/console/",
+    "build:local": "ng build --base-href=/default/console/",
+    "watch": "ng build --watch --configuration development",
+    "test": "ng test --browsers=ChromeHeadless --watch=false",
+    "run:dev": "",
+    "start:dev": "concurrently \"./../gradlew :core:runTestServer\" \"ng serve --proxy-config dev-proxy.config.json\""
+  },
+  "private": true,
+  "dependencies": {
+    "@angular/animations": "^15.2.2",
+    "@angular/cdk": "^15.2.2",
+    "@angular/common": "^15.2.2",
+    "@angular/compiler": "^15.2.2",
+    "@angular/core": "^15.2.2",
+    "@angular/forms": "^15.2.2",
+    "@angular/material": "^15.2.2",
+    "@angular/platform-browser": "^15.2.2",
+    "@angular/platform-browser-dynamic": "^15.2.2",
+    "@angular/router": "^15.2.2",
+    "rxjs": "~7.5.0",
+    "tslib": "^2.3.0",
+    "zone.js": "~0.11.4"
+  },
+  "devDependencies": {
+    "@angular-devkit/build-angular": "^15.2.4",
+    "@angular/cli": "~15.2.4",
+    "@angular/compiler-cli": "^15.2.2",
+    "@types/jasmine": "~4.0.0",
+    "@types/node": "^18.11.18",
+    "concurrently": "^7.6.0",
+    "jasmine-core": "~4.3.0",
+    "karma": "~6.4.0",
+    "karma-chrome-launcher": "~3.1.0",
+    "karma-coverage": "~2.2.0",
+    "karma-jasmine": "~5.1.0",
+    "karma-jasmine-html-reporter": "~2.0.0",
+    "typescript": "~4.9.4"
+  }
+}

console-webapp/src/app/app-routing.module.ts

@@ -0,0 +1,29 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { NgModule } from '@angular/core';
+import { RouterModule, Routes } from '@angular/router';
+import {TldsComponent} from './tlds/tlds.component';
+import {HomeComponent} from './home/home.component';
+
+const routes: Routes = [
+  { path: 'home', component: HomeComponent },
+  { path: 'tlds', component: TldsComponent },
+];
+
+@NgModule({
+  imports: [RouterModule.forRoot(routes)],
+  exports: [RouterModule]
+})
+export class AppRoutingModule { }

console-webapp/src/app/app.component.html

@@ -0,0 +1,14 @@
+<div class="toolbar" role="banner">
+  Nomulus Console
+</div>
+
+<div class="content" role="main">
+  <nav>
+    <ul>
+      <li><a routerLink="/home" routerLinkActive="active" ariaCurrentWhenActive="page">Home page</a></li>
+      <li><a routerLink="/tlds" routerLinkActive="active" ariaCurrentWhenActive="page">TLDs</a></li>
+    </ul>
+  </nav>
+</div>
+
+<router-outlet></router-outlet>

console-webapp/src/app/app.component.less

@@ -0,0 +1,35 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+:host {
+  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
+  font-size: 14px;
+  color: #333;
+  box-sizing: border-box;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  margin: 8px 0;
+}
+
+p {
+  margin: 0;
+}

console-webapp/src/app/app.component.spec.ts

@@ -0,0 +1,37 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { TestBed } from '@angular/core/testing';
+import { RouterTestingModule } from '@angular/router/testing';
+import { AppComponent } from './app.component';
+
+describe('AppComponent', () => {
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [
+        RouterTestingModule
+      ],
+      declarations: [
+        AppComponent
+      ],
+    }).compileComponents();
+  });
+
+  it('should create the app', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app).toBeTruthy();
+  });
+
+});

console-webapp/src/app/app.component.ts

@@ -1,4 +1,4 @@
-// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.tools;
+import { Component } from '@angular/core';
 
-/**
- * Marker interface for commands that use the remote api.
- *
- * <p>Just implementing this is sufficient to use the remote api; {@link RegistryTool} will install
- * it as needed.
- */
-public interface CommandWithRemoteApi extends Command {}
+@Component({
+  selector: 'app-root',
+  templateUrl: './app.component.html',
+  styleUrls: ['./app.component.less']
+})
+export class AppComponent {
+
+}

console-webapp/src/app/app.module.ts

@@ -0,0 +1,41 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { NgModule } from '@angular/core';
+import { BrowserModule } from '@angular/platform-browser';
+
+import { AppRoutingModule } from './app-routing.module';
+import { AppComponent } from './app.component';
+import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
+import {MaterialModule} from './material.module';
+
+import { HomeComponent } from './home/home.component';
+import { TldsComponent } from './tlds/tlds.component';
+
+@NgModule({
+  declarations: [
+    AppComponent,
+    HomeComponent,
+    TldsComponent,
+  ],
+  imports: [
+    MaterialModule,
+    BrowserModule,
+    AppRoutingModule,
+    BrowserAnimationsModule
+  ],
+  providers: [],
+  bootstrap: [AppComponent]
+})
+export class AppModule { }

console-webapp/src/app/home/home.component.html

@@ -0,0 +1,14 @@
+<h3>Recent Activity</h3>
+<table mat-table [dataSource]="dataSource" class="mat-elevation-z8 console-home__activity">
+  <ng-container *ngFor="let column of columns" [matColumnDef]="column.columnDef">
+    <th mat-header-cell *matHeaderCellDef>
+      {{column.header}}
+    </th>
+    <td mat-cell *matCellDef="let row">
+      {{column.cell(row)}}
+    </td>
+  </ng-container>
+
+  <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+  <tr mat-row *matRowDef="let row; columns: displayedColumns;"></tr>
+</table>

console-webapp/src/app/home/home.component.less

@@ -0,0 +1,14 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+

console-webapp/src/app/home/home.component.spec.ts

@@ -0,0 +1,39 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { HomeComponent } from './home.component';
+import {MaterialModule} from '../material.module';
+
+describe('HomeComponent', () => {
+  let component: HomeComponent;
+  let fixture: ComponentFixture<HomeComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [MaterialModule],
+      declarations: [ HomeComponent ]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(HomeComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

console-webapp/src/app/home/home.component.ts

@@ -0,0 +1,94 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Component } from '@angular/core';
+
+export interface ActivityRecord {
+  eventType: string;
+  userName: string;
+  registrarName: string;
+  timestamp: string;
+  details: string
+}
+
+const MOCK_DATA: ActivityRecord[] = [
+  {eventType: "Export DUMS", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Update Contact", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Delete Domain", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Export DUMS", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Update Contact", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Delete Domain", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Export DUMS", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Update Contact", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Delete Domain", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Export DUMS", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Update Contact", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Delete Domain", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Export DUMS", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Update Contact", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Delete Domain", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Export DUMS", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Update Contact", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Delete Domain", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Export DUMS", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Update Contact", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Delete Domain", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Export DUMS", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Update Contact", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+  {eventType: "Delete Domain", userName:"user3", registrarName: "registrar1", timestamp: "2022-03-15T19:46:39.007", details: "All Domains under management exported as .csv file" },
+];
+
+
+@Component({
+  selector: 'app-home',
+  templateUrl: './home.component.html',
+  styleUrls: ['./home.component.less']
+})
+export class HomeComponent {
+
+  columns = [
+    {
+      columnDef: 'eventType',
+      header: 'Event Type',
+      cell:(record: ActivityRecord) => `${record.eventType}`,
+    },
+    {
+      columnDef: 'userName',
+      header: 'User',
+      cell: (record: ActivityRecord)  => `${record.userName}`,
+    },
+    {
+      columnDef: 'registrarName',
+      header: 'Registrar',
+      cell: (record: ActivityRecord)  => `${record.registrarName}`,
+    },
+    {
+      columnDef: 'timestamp',
+      header: 'Timestamp',
+      cell: (record: ActivityRecord)  => `${record.timestamp}`,
+    },
+    {
+      columnDef: 'details',
+      header: 'Details',
+      cell: (record: ActivityRecord)  => `${record.details}`,
+    },
+  ];
+  dataSource = MOCK_DATA;
+  displayedColumns = this.columns.map(c => c.columnDef);
+
+  constructor() {
+
+
+  }
+}

console-webapp/src/app/material.module.ts

@@ -0,0 +1,26 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {NgModule} from '@angular/core';
+import {MatCardModule} from '@angular/material/card';
+import {MatTableModule} from '@angular/material/table';
+
+const MATERIAL_MODULES = [
+  MatCardModule,
+  MatTableModule,
+];
+
+@NgModule({imports: MATERIAL_MODULES, exports: MATERIAL_MODULES})
+export class MaterialModule {
+}

console-webapp/src/app/tlds/tlds.component.html

@@ -0,0 +1,24 @@
+
+<div class="console-tlds__cards">
+  <mat-card class="console-tlds__card">
+    <mat-card-title>.how</mat-card-title>
+    <mat-card-subtitle>A place for thinkers, tinkerers, and knowledge seekers</mat-card-subtitle>
+    <mat-card-actions class="console-tlds__card-links">
+      <a title="Onboarding Now" href="#" target="_blank" rel="noopener">Onboarding Now</a>
+      <a title="Marketing Materials" href="#" target="_blank" rel="noopener">Marketing Materials</a>
+      <a title="Visit get.how for more information" href="#" target="_blank" rel="noopener">Visit get.how for more information</a>
+    </mat-card-actions>
+  </mat-card>
+</div>
+
+<div class="console-tlds__cards">
+  <mat-card class="console-tlds__card">
+    <mat-card-title>.soy</mat-card-title>
+    <mat-card-subtitle>A place for thinkers, tinkerers, and knowledge seekers</mat-card-subtitle>
+    <mat-card-actions class="console-tlds__card-links">
+      <a title="Onboarding Now" href="#" target="_blank" rel="noopener">Onboarding Now</a>
+      <a title="Marketing Materials" href="#" target="_blank" rel="noopener">Marketing Materials</a>
+      <a title="Visit get.how for more information" href="#" target="_blank" rel="noopener">Visit iam.soy for more information</a>
+    </mat-card-actions>
+  </mat-card>
+</div>

console-webapp/src/app/tlds/tlds.component.less

@@ -0,0 +1,28 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+.console-tlds {
+  &__cards {
+    display: flex;
+    border-top: 1px solid #ddd;
+    padding: 1rem;
+  }
+  &__card {
+    max-width: 300px;
+  }
+  &__card-links {
+    display: flex;
+    flex-direction: column;
+  }
+}

console-webapp/src/app/tlds/tlds.component.spec.ts

@@ -0,0 +1,39 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { TldsComponent } from './tlds.component';
+import {MaterialModule} from '../material.module';
+
+describe('TldsComponent', () => {
+  let component: TldsComponent;
+  let fixture: ComponentFixture<TldsComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [MaterialModule],
+      declarations: [ TldsComponent ]
+    })
+    .compileComponents();
+
+    fixture = TestBed.createComponent(TldsComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

console-webapp/src/app/tlds/tlds.component.ts

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,16 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+import { Component, OnInit } from '@angular/core';
 
-import com.google.appengine.api.datastore.Key;
-import java.util.Optional;
+@Component({
+  selector: 'app-tlds',
+  templateUrl: './tlds.component.html',
+  styleUrls: ['./tlds.component.less']
+})
+export class TldsComponent implements OnInit {
 
-/** Utility methods for working with the App Engine Datastore service. */
-public class DatastoreServiceUtils {
+  constructor() { }
 
-  /** Returns the name or id of a key, which may be a string or a long. */
-  public static Object getNameOrId(Key key) {
-    return Optional.<Object>ofNullable(key.getName()).orElse(key.getId());
+  ngOnInit(): void {
   }
+
 }

console-webapp/src/environments/environment.prod.ts

@@ -0,0 +1,17 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export const environment = {
+  production: true
+};

console-webapp/src/environments/environment.ts

@@ -0,0 +1,30 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file can be replaced during build by using the `fileReplacements` array.
+// `ng build` replaces `environment.ts` with `environment.prod.ts`.
+// The list of file replacements can be found in `angular.json`.
+
+export const environment = {
+  production: false
+};
+
+/*
+ * For easier debugging in development mode, you can import the following file
+ * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`.
+ *
+ * This import should be commented out in production mode because it will have a negative impact
+ * on performance if an error is thrown.
+ */
+// import 'zone.js/plugins/zone-error';  // Included with Angular CLI.

console-webapp/src/index.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>Nomulus Console</title>
+  <base href="/">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <link rel="icon" type="image/x-icon" href="favicon.ico">
+  <link rel="preconnect" href="https://fonts.gstatic.com">
+  <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500&display=swap" rel="stylesheet">
+  <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+</head>
+<body class="mat-typography">
+  <app-root></app-root>
+</body>
+</html>

console-webapp/src/main.ts

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,23 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+import { enableProdMode } from '@angular/core';
+import { platformBrowserDynamic } from '@angular/platform-browser-dynamic';
 
-import java.io.Serializable;
+import { AppModule } from './app/app.module';
+import { environment } from './environments/environment';
 
-/** Used to query whether requests are still running. */
-public interface RequestStatusChecker extends Serializable {
-
-  /**
-   * Returns the unique log identifier of the current request.
-   *
-   * <p>Multiple calls must return the same value during the same Request.
-   */
-  String getLogId();
-
-  /**
-   * Returns true if the given request is currently running.
-   */
-  boolean isRunning(String requestLogId);
+if (environment.production) {
+  enableProdMode();
 }
 
+platformBrowserDynamic().bootstrapModule(AppModule)
+  .catch(err => console.error(err));

console-webapp/src/polyfills.ts

@@ -0,0 +1,53 @@
+/**
+ * This file includes polyfills needed by Angular and is loaded before the app.
+ * You can add your own extra polyfills to this file.
+ *
+ * This file is divided into 2 sections:
+ *   1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers.
+ *   2. Application imports. Files imported after ZoneJS that should be loaded before your main
+ *      file.
+ *
+ * The current setup is for so-called "evergreen" browsers; the last versions of browsers that
+ * automatically update themselves. This includes recent versions of Safari, Chrome (including
+ * Opera), Edge on the desktop, and iOS and Chrome on mobile.
+ *
+ * Learn more in https://angular.io/guide/browser-support
+ */
+
+/***************************************************************************************************
+ * BROWSER POLYFILLS
+ */
+
+/**
+ * By default, zone.js will patch all possible macroTask and DomEvents
+ * user can disable parts of macroTask/DomEvents patch by setting following flags
+ * because those flags need to be set before `zone.js` being loaded, and webpack
+ * will put import in the top of bundle, so user need to create a separate file
+ * in this directory (for example: zone-flags.ts), and put the following flags
+ * into that file, and then add the following code before importing zone.js.
+ * import './zone-flags';
+ *
+ * The flags allowed in zone-flags.ts are listed here.
+ *
+ * The following flags will work for all browsers.
+ *
+ * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame
+ * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick
+ * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames
+ *
+ *  in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js
+ *  with the following flag, it will bypass `zone.js` patch for IE/Edge
+ *
+ *  (window as any).__Zone_enable_cross_context_check = true;
+ *
+ */
+
+/***************************************************************************************************
+ * Zone JS is required by default for Angular itself.
+ */
+import 'zone.js';  // Included with Angular CLI.
+
+
+/***************************************************************************************************
+ * APPLICATION IMPORTS
+ */

console-webapp/src/styles.less

@@ -0,0 +1,18 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+
+html, body { height: 100%; }
+body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; }

console-webapp/src/test.ts

@@ -0,0 +1,14 @@
+// This file is required by karma.conf.js and loads recursively all the .spec and framework files
+
+import 'zone.js/testing';
+import { getTestBed } from '@angular/core/testing';
+import {
+  BrowserDynamicTestingModule,
+  platformBrowserDynamicTesting
+} from '@angular/platform-browser-dynamic/testing';
+
+// First, initialize the Angular testing environment.
+getTestBed().initTestEnvironment(
+  BrowserDynamicTestingModule,
+  platformBrowserDynamicTesting(),
+);

console-webapp/tsconfig.app.json

@@ -0,0 +1,15 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/app",
+    "types": []
+  },
+  "files": [
+    "src/main.ts",
+    "src/polyfills.ts"
+  ],
+  "include": [
+    "src/**/*.d.ts"
+  ]
+}

console-webapp/tsconfig.json

@@ -0,0 +1,33 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "compileOnSave": false,
+  "compilerOptions": {
+    "baseUrl": "./",
+    "outDir": "./dist/out-tsc",
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "noImplicitOverride": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "sourceMap": true,
+    "declaration": false,
+    "downlevelIteration": true,
+    "experimentalDecorators": true,
+    "moduleResolution": "node",
+    "importHelpers": true,
+    "target": "ES2022",
+    "module": "es2020",
+    "lib": [
+      "es2020",
+      "dom"
+    ],
+    "useDefineForClassFields": false
+  },
+  "angularCompilerOptions": {
+    "enableI18nLegacyMessageIdFormat": false,
+    "strictInjectionParameters": true,
+    "strictInputAccessModifiers": true,
+    "strictTemplates": true
+  }
+}

console-webapp/tsconfig.spec.json

@@ -0,0 +1,18 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/spec",
+    "types": [
+      "jasmine"
+    ]
+  },
+  "files": [
+    "src/test.ts",
+    "src/polyfills.ts"
+  ],
+  "include": [
+    "src/**/*.spec.ts",
+    "src/**/*.d.ts"
+  ]
+}

core/build.gradle

@@ -68,8 +68,6 @@ def dockerIncompatibleTestPatterns = [
 // Nomulus classes, e.g., threads and objects retained by frameworks.
 // TODO(weiminyu): identify cause and fix offending tests.
 def fragileTestPatterns = [
-    // Test Datastore inexplicably aborts transaction.
-    "google/registry/model/tmch/ClaimsListShardTest.*",
     // Changes cache timeouts and for some reason appears to have contention
     // with other tests.
     "google/registry/whois/WhoisCommandFactoryTest.*",
@@ -163,17 +161,11 @@ configurations {
 dependencies {
   def deps = rootProject.dependencyMap
 
-  // Custom-built objectify jar at commit ecd5165, included in Nomulus
-  // release.
-  implementation files(
-      "${rootDir}/third_party/objectify/v4_1/objectify-4.1.3.jar")
-
   testRuntimeOnly files(sourceSets.test.resources.srcDirs)
 
   implementation deps['com.beust:jcommander']
   implementation deps['com.github.ben-manes.caffeine:caffeine']
   implementation deps['com.google.api:gax']
-  implementation deps['com.google.api.grpc:proto-google-cloud-datastore-v1']
   implementation deps['com.google.api.grpc:proto-google-common-protos']
   implementation deps['com.google.api.grpc:proto-google-cloud-secretmanager-v1']
   implementation deps['com.google.api-client:google-api-client']
@@ -194,13 +186,9 @@ dependencies {
   implementation deps['com.google.apis:google-api-services-sheets']
   implementation deps['com.google.apis:google-api-services-storage']
   testImplementation deps['com.google.appengine:appengine-api-stubs']
-  implementation deps['com.google.appengine.tools:appengine-gcs-client']
-  implementation deps['com.google.appengine.tools:appengine-pipeline']
-  implementation deps['com.google.appengine:appengine-remote-api']
   implementation deps['com.google.auth:google-auth-library-credentials']
   implementation deps['com.google.auth:google-auth-library-oauth2-http']
   implementation deps['com.google.cloud.bigdataoss:util']
-  implementation deps['com.google.cloud.datastore:datastore-v1-proto-client']
   implementation deps['com.google.cloud.sql:jdbc-socket-factory-core']
   runtimeOnly deps['com.google.cloud.sql:postgres-socket-factory']
   implementation deps['com.google.cloud:google-cloud-secretmanager']
@@ -278,6 +266,8 @@ dependencies {
   implementation deps['org.jsoup:jsoup']
   testImplementation deps['org.mortbay.jetty:jetty']
   implementation deps['org.postgresql:postgresql']
+  implementation "org.eclipse.jetty:jetty-server:9.4.49.v20220914"
+  implementation "org.eclipse.jetty:jetty-servlet:9.4.49.v20220914"
   testImplementation deps['org.seleniumhq.selenium:selenium-api']
   testImplementation deps['org.seleniumhq.selenium:selenium-chrome-driver']
   testImplementation deps['org.seleniumhq.selenium:selenium-java']
@@ -703,10 +693,6 @@ createToolTask(
     'google.registry.tools.DevTool',
     sourceSets.nonprod)
 
-createToolTask(
-    'createSyntheticDomainHistories',
-    'google.registry.tools.javascrap.CreateSyntheticDomainHistoriesPipeline')
-
 project.tasks.create('generateSqlSchema', JavaExec) {
   classpath = sourceSets.nonprod.runtimeClasspath
   main = 'google.registry.tools.DevTool'
@@ -758,9 +744,14 @@ if (environment == 'alpha') {
           ],
       invoicing          :
           [
-              mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
+              mainClass: 'google.registry.beam.billing.InvoicingPipeline',
               metaData : 'google/registry/beam/invoicing_pipeline_metadata.json'
           ],
+      expandBilling      :
+          [
+              mainClass: 'google.registry.beam.billing.ExpandBillingRecurrencesPipeline',
+              metaData : 'google/registry/beam/expand_billing_recurrences_pipeline_metadata.json'
+          ],
       rde                :
           [
               mainClass: 'google.registry.beam.rde.RdePipeline',
@@ -771,6 +762,11 @@ if (environment == 'alpha') {
               mainClass: 'google.registry.beam.resave.ResaveAllEppResourcesPipeline',
               metaData: 'google/registry/beam/resave_all_epp_resources_pipeline_metadata.json'
           ],
+      wipeOutContactHistoryPii:
+              [
+                      mainClass: 'google.registry.beam.wipeout.WipeOutContactHistoryPiiPipeline',
+                      metaData: 'google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json'
+              ],
   ]
   project.tasks.create("stageBeamPipelines") {
     doLast {
@@ -1033,6 +1029,7 @@ test {
   // TODO(weiminyu): Remove dependency on sqlIntegrationTest
 }.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
 
+
 // When we override tests, we also break the cleanTest command.
 cleanTest.dependsOn(cleanFragileTest, cleanOutcastTest, cleanStandardTest,
                     cleanRegistryToolIntegrationTest, cleanSqlIntegrationTest)

core/gradle.lockfile

@@ -8,29 +8,27 @@ args4j:args4j:2.0.26=css
 cglib:cglib-nodep:2.2=css
 com.101tec:zkclient:0.10=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.beust:jcommander:1.60=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.electronwill.night-config:core:3.6.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.electronwill.night-config:toml:3.6.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson.core:jackson-annotations:2.13.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson.core:jackson-core:2.13.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson.core:jackson-databind:2.13.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.13.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson.datatype:jackson-datatype-joda:2.13.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.fasterxml.jackson:jackson-bom:2.13.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-annotations:2.14.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-core:2.14.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.core:jackson-databind:2.14.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.14.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.14.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.datatype:jackson-datatype-joda:2.14.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.fasterxml.jackson:jackson-bom:2.14.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.fasterxml:classmate:1.5.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.ben-manes.caffeine:caffeine:2.7.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 com.github.ben-manes.caffeine:caffeine:2.9.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.docker-java:docker-java-api:3.2.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.docker-java:docker-java-transport-zerodep:3.2.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.docker-java:docker-java-transport:3.2.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.github.jnr:jffi:1.3.9=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.jnr:jffi:1.3.10=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.jnr:jnr-a64asm:1.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.github.jnr:jnr-constants:0.10.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.github.jnr:jnr-enxio:0.32.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.github.jnr:jnr-ffi:2.2.11=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.github.jnr:jnr-posix:3.1.15=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.github.jnr:jnr-unixsocket:0.38.17=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.jnr:jnr-constants:0.10.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.jnr:jnr-enxio:0.32.14=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.jnr:jnr-ffi:2.2.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.jnr:jnr-posix:3.1.16=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.jnr:jnr-unixsocket:0.38.19=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.jnr:jnr-x86asm:1.0.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 com.google.android:annotations:4.1.1.4=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
@@ -39,73 +37,71 @@ com.google.api-client:google-api-client-jackson2:1.32.2=compileClasspath,default
 com.google.api-client:google-api-client-java6:1.35.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api-client:google-api-client-servlet:1.35.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api-client:google-api-client:1.35.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:gapic-google-cloud-storage-v2:2.14.0-alpha=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.20.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.144.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.144.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:gapic-google-cloud-storage-v2:2.17.2-alpha=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.25.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.149.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.149.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:2.11.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.102.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.6.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.29.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.29.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-spanner-v1:6.29.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-cloud-storage-v2:2.14.0-alpha=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-common-protos:2.9.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:grpc-google-iam-v1:1.6.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.20.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.144.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.144.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.11.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-bigtable-v2:2.11.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.102.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-firestore-v1:3.4.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-monitoring-v3:3.4.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.102.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.6.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-secretmanager-v1:2.5.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:2.5.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.29.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.29.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-spanner-v1:6.29.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-storage-v2:2.14.0-alpha=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-tasks-v2:2.5.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.95.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.95.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-common-protos:2.9.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api.grpc:proto-google-iam-v1:1.6.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api:api-common:2.2.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api:gax-grpc:2.19.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api:gax-httpjson:0.104.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.api:gax:2.19.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:2.16.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.103.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.33.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.33.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.33.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-cloud-storage-v2:2.17.2-alpha=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:grpc-google-common-protos:2.10.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.25.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.149.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.149.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.16.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.16.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.103.5=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.7.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-monitoring-v3:1.64.0=compileClasspath,nonprodCompileClasspath,testCompileClasspath
+com.google.api.grpc:proto-google-cloud-monitoring-v3:3.6.0=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.103.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-secretmanager-v1:2.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:2.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.33.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.33.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.33.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-storage-v2:2.17.2-alpha=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-tasks-v2:2.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.99.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.99.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-common-protos:2.13.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api.grpc:proto-google-iam-v1:1.8.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api:api-common:2.5.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api:gax-grpc:2.22.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api:gax-httpjson:0.107.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.api:gax:2.22.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.apis:google-api-services-appengine:v1-rev20220818-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.apis:google-api-services-bigquery:v2-rev20220827-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.apis:google-api-services-appengine:v1-rev20230109-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.apis:google-api-services-bigquery:v2-rev20220924-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-clouddebugger:v2-rev20220318-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-cloudresourcemanager:v1-rev20220828-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.apis:google-api-services-dataflow:v1b3-rev20220812-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.apis:google-api-services-dataflow:v1b3-rev20220920-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-groupssettings:v1-rev20210624-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-healthcare:v1-rev20220818-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.apis:google-api-services-monitoring:v3-rev20220930-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.apis:google-api-services-pubsub:v1-rev20220829-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.apis:google-api-services-sheets:v4-rev20220927-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.apis:google-api-services-sqladmin:v1beta4-rev20221017-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.apis:google-api-services-monitoring:v3-rev20230123-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.apis:google-api-services-pubsub:v1-rev20220904-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.apis:google-api-services-sheets:v4-rev20221216-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.apis:google-api-services-sqladmin:v1beta4-rev20230111-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.appengine.tools:appengine-gcs-client:0.8.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.appengine.tools:appengine-pipeline:0.2.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.appengine:appengine-api-1.0-sdk:2.0.9=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.appengine:appengine-api-stubs:2.0.9=testCompileClasspath,testRuntimeClasspath
-com.google.appengine:appengine-remote-api:2.0.9=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.appengine:appengine-api-1.0-sdk:1.9.86=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath
+com.google.appengine:appengine-api-1.0-sdk:2.0.10=testCompileClasspath,testRuntimeClasspath
+com.google.appengine:appengine-api-stubs:2.0.10=testCompileClasspath,testRuntimeClasspath
 com.google.appengine:appengine-testing:1.9.86=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.auth:google-auth-library-credentials:1.12.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.auth:google-auth-library-oauth2-http:1.12.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.auth:google-auth-library-credentials:1.14.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.auth:google-auth-library-oauth2-http:1.14.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor,compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.auto.service:auto-service:1.0.1=annotationProcessor
-com.google.auto.value:auto-value-annotations:1.10=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.auto.value:auto-value:1.10=annotationProcessor,default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testRuntimeClasspath
+com.google.auto.value:auto-value-annotations:1.10.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.auto.value:auto-value:1.10.1=annotationProcessor,compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
 com.google.auto:auto-common:0.10=errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 com.google.auto:auto-common:1.2=annotationProcessor
 com.google.closure-stylesheets:closure-stylesheets:1.5.0=css
@@ -114,38 +110,39 @@ com.google.cloud.bigdataoss:util:2.2.6=compileClasspath,default,deploy_jar,nonpr
 com.google.cloud.bigtable:bigtable-client-core:1.26.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.cloud.bigtable:bigtable-metrics-api:1.26.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.cloud.datastore:datastore-v1-proto-client:2.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud.sql:jdbc-socket-factory-core:1.7.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud.sql:postgres-socket-factory:1.7.2=default,deploy_jar,runtimeClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-bigquerystorage:2.20.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-bigtable-stats:2.11.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-bigtable:2.11.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-core-grpc:2.8.22=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-core-http:2.8.22=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-core:2.8.22=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-firestore:3.4.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-monitoring:3.4.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-nio:0.124.19=testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-pubsub:1.120.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-pubsublite:1.6.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-secretmanager:2.5.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-spanner:6.29.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-storage:2.14.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:google-cloud-tasks:2.5.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:grpc-gcp:1.2.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.4.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud.sql:jdbc-socket-factory-core:1.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud.sql:postgres-socket-factory:1.9.0=default,deploy_jar,runtimeClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-bigquerystorage:2.25.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-bigtable-stats:2.16.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-bigtable:2.16.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-core-grpc:2.9.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-core-http:2.9.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-core:2.9.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-firestore:3.7.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-monitoring:1.82.0=compileClasspath,nonprodCompileClasspath,testCompileClasspath
+com.google.cloud:google-cloud-monitoring:3.6.0=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-nio:0.126.3=testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-pubsub:1.121.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-pubsublite:1.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-secretmanager:2.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-spanner:6.33.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-storage:2.17.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:google-cloud-tasks:2.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:grpc-gcp:1.3.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.7.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.code.findbugs:jFormatString:3.0.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 com.google.code.findbugs:jsr305:3.0.1=css
 com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,checkstyle,compileClasspath,default,deploy_jar,errorprone,nonprodAnnotationProcessor,nonprodCompileClasspath,nonprodRuntime,nonprodRuntimeClasspath,runtime,runtimeClasspath,soy,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
-com.google.code.gson:gson:2.10=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.code.gson:gson:2.10.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.code.gson:gson:2.7=css,soy
 com.google.common.html.types:types:1.0.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,soy,testCompileClasspath,testRuntimeClasspath
-com.google.dagger:dagger-compiler:2.44=annotationProcessor,testAnnotationProcessor
-com.google.dagger:dagger-producers:2.44=annotationProcessor,testAnnotationProcessor
-com.google.dagger:dagger-spi:2.44=annotationProcessor,testAnnotationProcessor
-com.google.dagger:dagger:2.44=annotationProcessor,compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
+com.google.dagger:dagger-compiler:2.44.2=annotationProcessor,testAnnotationProcessor
+com.google.dagger:dagger-producers:2.44.2=annotationProcessor,testAnnotationProcessor
+com.google.dagger:dagger-spi:2.44.2=annotationProcessor,testAnnotationProcessor
+com.google.dagger:dagger:2.44.2=annotationProcessor,compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
 com.google.devtools.ksp:symbol-processing-api:1.7.0-1.0.6=annotationProcessor,testAnnotationProcessor
 com.google.errorprone:error_prone_annotation:2.3.4=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
-com.google.errorprone:error_prone_annotations:2.16=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.errorprone:error_prone_annotations:2.18.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.errorprone:error_prone_annotations:2.3.4=checkstyle,errorprone,nonprodAnnotationProcessor,soy
 com.google.errorprone:error_prone_annotations:2.7.1=annotationProcessor,testAnnotationProcessor
 com.google.errorprone:error_prone_check_api:2.3.4=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
@@ -168,10 +165,10 @@ com.google.guava:guava:31.0.1-jre=annotationProcessor,testAnnotationProcessor
 com.google.guava:guava:31.1-jre=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,checkstyle,compileClasspath,default,deploy_jar,errorprone,nonprodAnnotationProcessor,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,soy,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath
 com.google.gwt:gwt-user:2.10.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-apache-v2:1.42.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-appengine:1.42.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-apache-v2:1.42.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-appengine:1.42.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.http-client:google-http-client-gson:1.42.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.http-client:google-http-client-jackson2:1.42.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.http-client:google-http-client-jackson2:1.42.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.http-client:google-http-client-protobuf:1.41.8=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.http-client:google-http-client:1.42.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.inject.extensions:guice-multibindings:4.1.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,soy,testCompileClasspath,testRuntimeClasspath
@@ -192,9 +189,9 @@ com.google.oauth-client:google-oauth-client-java6:1.34.1=compileClasspath,defaul
 com.google.oauth-client:google-oauth-client-jetty:1.34.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.oauth-client:google-oauth-client-servlet:1.34.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.oauth-client:google-oauth-client:1.34.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-com.google.protobuf:protobuf-java-util:3.21.8=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.protobuf:protobuf-java-util:3.21.12=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.protobuf:protobuf-java:2.5.0=css
-com.google.protobuf:protobuf-java:3.21.8=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.google.protobuf:protobuf-java:3.21.12=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.google.protobuf:protobuf-java:3.4.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 com.google.protobuf:protobuf-java:4.0.0-rc-2=soy
 com.google.re2j:re2j:1.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -211,6 +208,7 @@ com.puppycrawl.tools:checkstyle:8.37=checkstyle
 com.squareup.okhttp3:okhttp:3.11.0=testCompileClasspath,testRuntimeClasspath
 com.squareup.okio:okio:1.14.0=testCompileClasspath,testRuntimeClasspath
 com.squareup:javapoet:1.13.0=annotationProcessor,testAnnotationProcessor
+com.squareup:kotlinpoet:1.11.0=annotationProcessor,testAnnotationProcessor
 com.sun.activation:jakarta.activation:1.2.2=jaxb
 com.sun.activation:javax.activation:1.2.0=jaxb
 com.sun.istack:istack-commons-runtime:3.0.7=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -226,7 +224,7 @@ commons-beanutils:commons-beanutils:1.9.4=checkstyle
 commons-codec:commons-codec:1.15=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 commons-collections:commons-collections:3.2.2=checkstyle
 commons-logging:commons-logging:1.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-dnsjava:dnsjava:3.5.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+dnsjava:dnsjava:3.5.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 guru.nidi.com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0=nonprodRuntime,runtime,testRuntimeClasspath
 guru.nidi.com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0=nonprodRuntime,runtime,testRuntimeClasspath
 guru.nidi.com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0=nonprodRuntime,runtime,testRuntimeClasspath
@@ -242,39 +240,37 @@ io.confluent:kafka-schema-registry-client:5.3.2=compileClasspath,default,deploy_
 io.dropwizard.metrics:metrics-core:3.1.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.github.classgraph:classgraph:4.8.104=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.github.java-diff-utils:java-diff-utils:4.12=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-alts:1.48.0=compileClasspath,nonprodCompileClasspath,testCompileClasspath
-io.grpc:grpc-alts:1.50.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.grpc:grpc-api:1.50.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-auth:1.50.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-census:1.48.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-context:1.50.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-core:1.50.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-googleapis:1.50.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.grpc:grpc-grpclb:1.48.0=compileClasspath,nonprodCompileClasspath,testCompileClasspath
-io.grpc:grpc-grpclb:1.50.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.grpc:grpc-netty-shaded:1.48.0=compileClasspath,nonprodCompileClasspath,testCompileClasspath
-io.grpc:grpc-netty-shaded:1.50.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.grpc:grpc-netty:1.48.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-protobuf-lite:1.50.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-protobuf:1.50.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-services:1.48.0=compileClasspath,nonprodCompileClasspath,testCompileClasspath
-io.grpc:grpc-services:1.50.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.grpc:grpc-stub:1.50.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.grpc:grpc-xds:1.48.0=compileClasspath,nonprodCompileClasspath,testCompileClasspath
-io.grpc:grpc-xds:1.50.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.netty:netty-buffer:4.1.77.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.netty:netty-codec-http2:4.1.77.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.netty:netty-codec-http:4.1.77.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.netty:netty-codec-socks:4.1.77.Final=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.netty:netty-codec:4.1.77.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.netty:netty-common:4.1.77.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.netty:netty-handler-proxy:4.1.77.Final=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.netty:netty-handler:4.1.77.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.netty:netty-resolver:4.1.77.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-alts:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-api:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-auth:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-census:1.50.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-context:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-core:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-googleapis:1.52.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+io.grpc:grpc-grpclb:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-netty-shaded:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-netty:1.50.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-protobuf-lite:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-protobuf:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-rls:1.50.2=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+io.grpc:grpc-services:1.50.2=compileClasspath,nonprodCompileClasspath,testCompileClasspath
+io.grpc:grpc-services:1.52.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+io.grpc:grpc-stub:1.52.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.grpc:grpc-xds:1.50.2=compileClasspath,nonprodCompileClasspath,testCompileClasspath
+io.grpc:grpc-xds:1.52.1=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+io.netty:netty-buffer:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.netty:netty-codec-http2:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.netty:netty-codec-http:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.netty:netty-codec-socks:4.1.79.Final=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+io.netty:netty-codec:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.netty:netty-common:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.netty:netty-handler-proxy:4.1.79.Final=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+io.netty:netty-handler:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.netty:netty-resolver:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.netty:netty-tcnative-boringssl-static:2.0.52.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.netty:netty-tcnative-classes:2.0.52.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.netty:netty-transport-native-unix-common:4.1.77.Final=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
-io.netty:netty-transport:4.1.77.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.netty:netty-transport-native-unix-common:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+io.netty:netty-transport:4.1.79.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.opencensus:opencensus-api:0.31.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.opencensus:opencensus-contrib-exemplar-util:0.31.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.opencensus:opencensus-contrib-grpc-metrics:0.31.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -286,7 +282,7 @@ io.opencensus:opencensus-exporter-stats-stackdriver:0.31.0=compileClasspath,defa
 io.opencensus:opencensus-impl-core:0.31.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.opencensus:opencensus-impl:0.31.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 io.opencensus:opencensus-proto:0.2.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-io.perfmark:perfmark-api:0.25.0=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
+io.perfmark:perfmark-api:0.26.0=default,deploy_jar,nonprodRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
 jakarta.activation:jakarta.activation-api:2.1.0=jaxb,nonprodRuntime,runtime
 jakarta.xml.bind:jakarta.xml.bind-api:4.0.0=jaxb,nonprodRuntime,runtime
 javacc:javacc:4.1=css
@@ -306,8 +302,9 @@ jline:jline:1.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonp
 joda-time:joda-time:2.10.10=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 junit:junit:4.13.2=default,nonprodCompileClasspath,nonprodRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
 net.arnx:nashorn-promise:0.1.1=nonprodRuntime,runtime,testRuntimeClasspath
-net.bytebuddy:byte-buddy-agent:1.12.16=testCompileClasspath,testRuntimeClasspath
-net.bytebuddy:byte-buddy:1.12.18=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+net.bytebuddy:byte-buddy-agent:1.12.22=testCompileClasspath,testRuntimeClasspath
+net.bytebuddy:byte-buddy:1.12.18=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath
+net.bytebuddy:byte-buddy:1.12.22=testCompileClasspath,testRuntimeClasspath
 net.java.dev.javacc:javacc:4.1=css
 net.java.dev.jna:jna:5.8.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 net.ltgt.gradle.incap:incap:0.2=annotationProcessor,testAnnotationProcessor
@@ -317,38 +314,38 @@ org.apache.arrow:arrow-format:5.0.0=compileClasspath,default,deploy_jar,nonprodC
 org.apache.arrow:arrow-memory-core:5.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.arrow:arrow-vector:5.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.avro:avro:1.8.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-model-fn-execution:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-model-job-management:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-model-pipeline:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-runners-core-construction-java:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-runners-core-java:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-runners-direct-java:2.42.0=testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-runners-java-fn-execution:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-sdks-java-core:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-sdks-java-expansion-service:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-sdks-java-extensions-arrow:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-sdks-java-fn-execution:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.beam:beam-sdks-java-io-kafka:2.42.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-model-fn-execution:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-model-job-management:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-model-pipeline:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-runners-core-construction-java:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-runners-core-java:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-runners-direct-java:2.44.0=testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-runners-java-fn-execution:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-sdks-java-core:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-sdks-java-expansion-service:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-sdks-java-extensions-arrow:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-sdks-java-fn-execution:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.beam:beam-sdks-java-io-kafka:2.44.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.beam:beam-vendor-grpc-1_48_1:0.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.commons:commons-compress:1.21=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.commons:commons-compress:1.22=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-csv:1.9.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-exec:1.3=nonprodRuntime,runtime,testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-lang3:3.12.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.commons:commons-text:1.10.0=testCompileClasspath,testRuntimeClasspath
 org.apache.ftpserver:ftplet-api:1.2.0=testCompileClasspath,testRuntimeClasspath
 org.apache.ftpserver:ftpserver-core:1.2.0=testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpclient:4.5.13=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpcore:4.4.15=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpclient:4.5.14=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpcore:4.4.16=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.mina:mina-core:2.1.6=testCompileClasspath,testRuntimeClasspath
 org.apache.sshd:sshd-core:2.0.0=testCompileClasspath,testRuntimeClasspath
 org.apache.sshd:sshd-scp:2.0.0=testCompileClasspath,testRuntimeClasspath
 org.apache.sshd:sshd-sftp:2.0.0=testCompileClasspath,testRuntimeClasspath
-org.apache.tomcat:tomcat-annotations-api:10.1.1=testCompileClasspath,testRuntimeClasspath
+org.apache.tomcat:tomcat-annotations-api:11.0.0-M1=testCompileClasspath,testRuntimeClasspath
 org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
 org.bouncycastle:bcpg-jdk15on:1.67=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.bouncycastle:bcpkix-jdk15on:1.67=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -358,7 +355,7 @@ org.checkerframework:checker-compat-qual:2.5.5=annotationProcessor,compileClassp
 org.checkerframework:checker-qual:2.11.1=checkstyle
 org.checkerframework:checker-qual:3.0.0=errorprone,nonprodAnnotationProcessor
 org.checkerframework:checker-qual:3.12.0=annotationProcessor,testAnnotationProcessor
-org.checkerframework:checker-qual:3.26.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.checkerframework:checker-qual:3.29.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.checkerframework:checker-qual:3.5.0=nonprodRuntime,runtime,soy
 org.checkerframework:dataflow:3.0.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 org.checkerframework:javacutil:3.0.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
@@ -369,7 +366,14 @@ org.codehaus.mojo:animal-sniffer-annotations:1.22=default,deploy_jar,nonprodRunt
 org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.easymock:easymock:3.0=css
 org.eclipse.angus:angus-activation:1.0.0=nonprodRuntime,runtime
-org.flywaydb:flyway-core:9.7.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-http:9.4.49.v20220914=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-io:9.4.49.v20220914=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-security:9.4.49.v20220914=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-server:9.4.49.v20220914=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-servlet:9.4.49.v20220914=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-util-ajax:9.4.49.v20220914=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.eclipse.jetty:jetty-util:9.4.49.v20220914=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.flywaydb:flyway-core:9.12.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-core:4.0.1=nonprodRuntime,runtime
 org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-runtime:4.0.1=nonprodRuntime,runtime
@@ -385,14 +389,15 @@ org.hamcrest:hamcrest:2.2=testCompileClasspath,testRuntimeClasspath
 org.hibernate.common:hibernate-commons-annotations:5.1.2.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.hibernate:hibernate-core:5.6.14.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.hibernate:hibernate-hikaricp:5.6.14.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.jacoco:org.jacoco.agent:0.8.6=jacocoAgent,jacocoAnt
-org.jacoco:org.jacoco.ant:0.8.6=jacocoAnt
-org.jacoco:org.jacoco.core:0.8.6=jacocoAnt
-org.jacoco:org.jacoco.report:0.8.6=jacocoAnt
+org.jacoco:org.jacoco.agent:0.8.7=jacocoAgent,jacocoAnt
+org.jacoco:org.jacoco.ant:0.8.7=jacocoAnt
+org.jacoco:org.jacoco.core:0.8.7=jacocoAnt
+org.jacoco:org.jacoco.report:0.8.7=jacocoAnt
 org.javassist:javassist:3.26.0-GA=checkstyle
 org.jboss.logging:jboss-logging:3.4.3.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.jboss.spec.javax.transaction:jboss-transaction-api_1.2_spec:1.1.1.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.jboss:jandex:2.4.2.Final=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.jetbrains.kotlin:kotlin-reflect:1.6.10=annotationProcessor,testAnnotationProcessor
 org.jetbrains.kotlin:kotlin-stdlib-common:1.7.0=annotationProcessor,testAnnotationProcessor
 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.7.0=annotationProcessor,testAnnotationProcessor
 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.7.0=annotationProcessor,testAnnotationProcessor
@@ -400,50 +405,50 @@ org.jetbrains.kotlin:kotlin-stdlib:1.7.0=annotationProcessor,testAnnotationProce
 org.jetbrains.kotlinx:kotlinx-metadata-jvm:0.5.0=annotationProcessor,testAnnotationProcessor
 org.jetbrains:annotations:13.0=annotationProcessor,testAnnotationProcessor
 org.jetbrains:annotations:17.0.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.joda:joda-money:1.0.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.joda:joda-money:1.0.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.json:json:20160212=soy
 org.json:json:20200518=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.jsoup:jsoup:1.15.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.junit-pioneer:junit-pioneer:1.7.1=testCompileClasspath,testRuntimeClasspath
-org.junit.jupiter:junit-jupiter-api:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.jupiter:junit-jupiter-engine:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.jupiter:junit-jupiter-migrationsupport:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.jupiter:junit-jupiter-params:5.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-commons:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-engine:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-launcher:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-runner:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-suite-api:1.9.1=testCompileClasspath,testRuntimeClasspath
-org.junit.platform:junit-platform-suite-commons:1.9.1=testRuntimeClasspath
-org.junit:junit-bom:5.9.1=testCompileClasspath,testRuntimeClasspath
+org.junit-pioneer:junit-pioneer:2.0.0-RC1=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-api:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-engine:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-migrationsupport:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-params:5.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-commons:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-engine:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-launcher:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-runner:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-suite-api:1.9.2=testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-suite-commons:1.9.2=testRuntimeClasspath
+org.junit:junit-bom:5.9.2=testCompileClasspath,testRuntimeClasspath
 org.jvnet.staxex:stax-ex:1.8=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.mockito:mockito-core:1.10.19=css
-org.mockito:mockito-core:4.8.1=testCompileClasspath,testRuntimeClasspath
-org.mockito:mockito-junit-jupiter:4.8.1=testCompileClasspath,testRuntimeClasspath
+org.mockito:mockito-core:5.0.0=testCompileClasspath,testRuntimeClasspath
+org.mockito:mockito-junit-jupiter:5.0.0=testCompileClasspath,testRuntimeClasspath
 org.mortbay.jetty:jetty-util:6.1.26=testCompileClasspath,testRuntimeClasspath
 org.mortbay.jetty:jetty:6.1.26=testCompileClasspath,testRuntimeClasspath
 org.objenesis:objenesis:2.1=css
-org.objenesis:objenesis:3.2=testRuntimeClasspath
+org.objenesis:objenesis:3.3=testRuntimeClasspath
 org.opentest4j:opentest4j:1.2.0=testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm-analysis:7.0=soy
-org.ow2.asm:asm-analysis:8.0.1=jacocoAnt
+org.ow2.asm:asm-analysis:9.1=jacocoAnt
 org.ow2.asm:asm-analysis:9.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm-commons:7.0=soy
-org.ow2.asm:asm-commons:8.0.1=jacocoAnt
+org.ow2.asm:asm-commons:9.1=jacocoAnt
 org.ow2.asm:asm-commons:9.2=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm-tree:7.0=soy
-org.ow2.asm:asm-tree:8.0.1=jacocoAnt
+org.ow2.asm:asm-tree:9.1=jacocoAnt
 org.ow2.asm:asm-tree:9.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm-util:7.0=soy
 org.ow2.asm:asm-util:9.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:7.0=soy
-org.ow2.asm:asm:8.0.1=jacocoAnt
+org.ow2.asm:asm:9.1=jacocoAnt
 org.ow2.asm:asm:9.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.pcollections:pcollections:2.1.2=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 org.plumelib:plume-util:1.0.6=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 org.plumelib:reflection-util:0.0.2=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
 org.plumelib:require-javadoc:0.1.0=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
-org.postgresql:postgresql:42.5.0=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntime,nonprodRuntimeClasspath,runtime,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.5.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntime,nonprodRuntimeClasspath,runtime,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.reflections:reflections:0.9.12=checkstyle
 org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.seleniumhq.selenium:selenium-api:3.141.59=testCompileClasspath,testRuntimeClasspath
@@ -460,23 +465,23 @@ org.slf4j:jcl-over-slf4j:1.7.30=nonprodRuntime,runtime,testRuntimeClasspath
 org.slf4j:jul-to-slf4j:1.7.30=nonprodRuntime,runtime,testRuntimeClasspath
 org.slf4j:slf4j-api:1.7.30=nonprodRuntime,runtime
 org.slf4j:slf4j-api:1.7.36=compileClasspath,nonprodCompileClasspath,nonprodRuntimeClasspath,testCompileClasspath
-org.slf4j:slf4j-api:2.0.3=default,deploy_jar,runtimeClasspath,testRuntimeClasspath
-org.slf4j:slf4j-jdk14:2.0.3=default,deploy_jar,runtimeClasspath,testRuntimeClasspath
+org.slf4j:slf4j-api:2.0.6=default,deploy_jar,runtimeClasspath,testRuntimeClasspath
+org.slf4j:slf4j-jdk14:2.0.6=default,deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.springframework:spring-core:5.3.18=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.springframework:spring-expression:5.3.18=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.springframework:spring-jcl:5.3.18=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.testcontainers:database-commons:1.17.5=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.testcontainers:jdbc:1.17.5=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.testcontainers:junit-jupiter:1.17.5=testCompileClasspath,testRuntimeClasspath
-org.testcontainers:postgresql:1.17.5=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.testcontainers:selenium:1.17.5=testCompileClasspath,testRuntimeClasspath
-org.testcontainers:testcontainers:1.17.5=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.threeten:threetenbp:1.6.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.testcontainers:database-commons:1.17.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.testcontainers:jdbc:1.17.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.testcontainers:junit-jupiter:1.17.6=testCompileClasspath,testRuntimeClasspath
+org.testcontainers:postgresql:1.17.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.testcontainers:selenium:1.17.6=testCompileClasspath,testRuntimeClasspath
+org.testcontainers:testcontainers:1.17.6=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.threeten:threetenbp:1.6.5=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.tukaani:xz:1.5=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.w3c.css:sac:1.3=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.webjars.npm:viz.js-graphviz-java:2.1.3=nonprodRuntime,runtime,testRuntimeClasspath
 org.xerial.snappy:snappy-java:1.1.8.4=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.yaml:snakeyaml:1.31=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.yaml:snakeyaml:1.33=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 us.fatehi:schemacrawler-api:16.10.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 us.fatehi:schemacrawler-diagram:16.10.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 us.fatehi:schemacrawler-tools:16.10.1=compileClasspath,default,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -25,7 +25,6 @@ import com.google.common.flogger.FluentLogger;
 import google.registry.model.EppResource;
 import google.registry.persistence.VKey;
 import google.registry.request.Action.Service;
-import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
@@ -86,6 +85,6 @@ public final class AsyncTaskEnqueuer {
     cloudTasksUtils.enqueue(
         QUEUE_ASYNC_ACTIONS,
         cloudTasksUtils.createPostTaskWithDelay(
-            ResaveEntityAction.PATH, Service.BACKEND.toString(), params, etaDuration));
+            ResaveEntityAction.PATH, Service.BACKEND, params, etaDuration));
   }
 }

core/src/main/java/google/registry/batch/BatchModule.java

@@ -17,7 +17,6 @@ package google.registry.batch;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
-import static google.registry.batch.CannedScriptExecutionAction.SCRIPT_PARAM;
 import static google.registry.request.RequestParameters.extractBooleanParameter;
 import static google.registry.request.RequestParameters.extractIntParameter;
 import static google.registry.request.RequestParameters.extractLongParameter;
@@ -105,10 +104,27 @@ public class BatchModule {
   }
 
   @Provides
-  @Parameter(ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME)
-  static Optional<DateTime> provideCursorTime(HttpServletRequest req) {
-    return extractOptionalDatetimeParameter(
-        req, ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME);
+  @Parameter(ExpandBillingRecurrencesAction.PARAM_START_TIME)
+  static Optional<DateTime> provideStartTime(HttpServletRequest req) {
+    return extractOptionalDatetimeParameter(req, ExpandBillingRecurrencesAction.PARAM_START_TIME);
+  }
+
+  @Provides
+  @Parameter(ExpandBillingRecurrencesAction.PARAM_END_TIME)
+  static Optional<DateTime> provideEndTime(HttpServletRequest req) {
+    return extractOptionalDatetimeParameter(req, ExpandBillingRecurrencesAction.PARAM_END_TIME);
+  }
+
+  @Provides
+  @Parameter(WipeOutContactHistoryPiiAction.PARAM_CUTOFF_TIME)
+  static Optional<DateTime> provideCutoffTime(HttpServletRequest req) {
+    return extractOptionalDatetimeParameter(req, WipeOutContactHistoryPiiAction.PARAM_CUTOFF_TIME);
+  }
+
+  @Provides
+  @Parameter(ExpandBillingRecurrencesAction.PARAM_ADVANCE_CURSOR)
+  static boolean provideAdvanceCursor(HttpServletRequest req) {
+    return extractBooleanParameter(req, ExpandBillingRecurrencesAction.PARAM_ADVANCE_CURSOR);
   }
 
   @Provides
@@ -122,11 +138,4 @@ public class BatchModule {
   static boolean provideIsDryRun(HttpServletRequest req) {
     return extractBooleanParameter(req, PARAM_DRY_RUN);
   }
-
-  // TODO(b/234424397): remove method after credential changes are rolled out.
-  @Provides
-  @Parameter(SCRIPT_PARAM)
-  static String provideScriptName(HttpServletRequest req) {
-    return extractRequiredParameter(req, SCRIPT_PARAM);
-  }
 }

core/src/main/java/google/registry/batch/CannedScriptExecutionAction.java

@@ -16,11 +16,9 @@ package google.registry.batch;
 
 import static google.registry.request.Action.Method.POST;
 
-import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.FluentLogger;
-import google.registry.batch.cannedscript.GroupsApiChecker;
+import google.registry.batch.cannedscript.CannedScripts;
 import google.registry.request.Action;
-import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
 import javax.inject.Inject;
 
@@ -35,7 +33,7 @@ import javax.inject.Inject;
  * <p>This action can be invoked using the Nomulus CLI command: {@code nomulus -e ${env} curl
  * --service BACKEND -X POST -u '/_dr/task/executeCannedScript?script=${script_name}'}
  */
-// TODO(b/234424397): remove class after credential changes are rolled out.
+// TODO(b/277239043): remove class after credential changes are rolled out.
 @Action(
     service = Action.Service.BACKEND,
     path = "/_dr/task/executeCannedScript",
@@ -45,29 +43,18 @@ import javax.inject.Inject;
 public class CannedScriptExecutionAction implements Runnable {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  static final String SCRIPT_PARAM = "script";
-
-  static final ImmutableMap<String, Runnable> SCRIPTS =
-      ImmutableMap.of("runGroupsApiChecks", GroupsApiChecker::runGroupsApiChecks);
-
-  private final String scriptName;
-
   @Inject
-  CannedScriptExecutionAction(@Parameter(SCRIPT_PARAM) String scriptName) {
-    logger.atInfo().log("Received request to run script %s", scriptName);
-    this.scriptName = scriptName;
+  CannedScriptExecutionAction() {
+    logger.atInfo().log("Received request to run scripts.");
   }
 
   @Override
   public void run() {
-    if (!SCRIPTS.containsKey(scriptName)) {
-      throw new IllegalArgumentException("Script not found:" + scriptName);
-    }
     try {
-      SCRIPTS.get(scriptName).run();
-      logger.atInfo().log("Finished running %s.", scriptName);
+      CannedScripts.runAllChecks();
+      logger.atInfo().log("Finished running scripts.");
     } catch (Throwable t) {
-      logger.atWarning().withCause(t).log("Error executing %s", scriptName);
+      logger.atWarning().withCause(t).log("Error executing scripts.");
       throw new RuntimeException("Execution failed.");
     }
   }

core/src/main/java/google/registry/batch/CheckPackagesComplianceAction.java

@@ -13,24 +13,26 @@
 // limitations under the License.
 package google.registry.batch;
 
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.primitives.Ints;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.PackagePromotion;
 import google.registry.model.registrar.Registrar;
-import google.registry.model.registrar.RegistrarPoc;
 import google.registry.request.Action;
 import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.ui.server.SendEmailUtils;
+import google.registry.util.Clock;
 import java.util.Optional;
 import javax.inject.Inject;
+import org.joda.time.Days;
 
 /**
  * An action that checks all {@link PackagePromotion} objects for compliance with their max create
@@ -45,93 +47,180 @@ public class CheckPackagesComplianceAction implements Runnable {
   public static final String PATH = "/_dr/task/checkPackagesCompliance";
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private final SendEmailUtils sendEmailUtils;
-  private final String packageCreateLimitEmailSubjectText;
-  private final String packageCreateLimitEmailBodyText;
+  private final Clock clock;
+  private final String packageCreateLimitEmailSubject;
+  private final String packageDomainLimitWarningEmailSubject;
+  private final String packageDomainLimitUpgradeEmailSubject;
+  private final String packageCreateLimitEmailBody;
+  private final String packageDomainLimitWarningEmailBody;
+  private final String packageDomainLimitUpgradeEmailBody;
   private final String registrySupportEmail;
+  private static final int THIRTY_DAYS = 30;
+  private static final int FORTY_DAYS = 40;
 
   @Inject
   public CheckPackagesComplianceAction(
       SendEmailUtils sendEmailUtils,
-      @Config("packageCreateLimitEmailSubjectText") String packageCreateLimitEmailSubjectText,
-      @Config("packageCreateLimitEmailBodyText") String packageCreateLimitEmailBodyText,
+      Clock clock,
+      @Config("packageCreateLimitEmailSubject") String packageCreateLimitEmailSubject,
+      @Config("packageDomainLimitWarningEmailSubject") String packageDomainLimitWarningEmailSubject,
+      @Config("packageDomainLimitUpgradeEmailSubject") String packageDomainLimitUpgradeEmailSubject,
+      @Config("packageCreateLimitEmailBody") String packageCreateLimitEmailBody,
+      @Config("packageDomainLimitWarningEmailBody") String packageDomainLimitWarningEmailBody,
+      @Config("packageDomainLimitUpgradeEmailBody") String packageDomainLimitUpgradeEmailBody,
       @Config("registrySupportEmail") String registrySupportEmail) {
     this.sendEmailUtils = sendEmailUtils;
-    this.packageCreateLimitEmailSubjectText = packageCreateLimitEmailSubjectText;
-    this.packageCreateLimitEmailBodyText = packageCreateLimitEmailBodyText;
+    this.clock = clock;
+    this.packageCreateLimitEmailSubject = packageCreateLimitEmailSubject;
+    this.packageDomainLimitWarningEmailSubject = packageDomainLimitWarningEmailSubject;
+    this.packageDomainLimitUpgradeEmailSubject = packageDomainLimitUpgradeEmailSubject;
+    this.packageCreateLimitEmailBody = packageCreateLimitEmailBody;
+    this.packageDomainLimitWarningEmailBody = packageDomainLimitWarningEmailBody;
+    this.packageDomainLimitUpgradeEmailBody = packageDomainLimitUpgradeEmailBody;
     this.registrySupportEmail = registrySupportEmail;
   }
 
   @Override
   public void run() {
-    tm().transact(
-            () -> {
-              ImmutableList<PackagePromotion> packages = tm().loadAllOf(PackagePromotion.class);
-              ImmutableList.Builder<PackagePromotion> packagesOverCreateLimit =
-                  new ImmutableList.Builder<>();
-              for (PackagePromotion packagePromo : packages) {
-                Long creates =
-                    (Long)
-                        jpaTm()
-                            .query(
-                                "SELECT COUNT(*) FROM DomainHistory WHERE current_package_token ="
-                                    + " :token AND modificationTime >= :lastBilling AND type ="
-                                    + " 'DOMAIN_CREATE'")
-                            .setParameter("token", packagePromo.getToken().getKey().toString())
-                            .setParameter(
-                                "lastBilling", packagePromo.getNextBillingDate().minusYears(1))
-                            .getSingleResult();
-                if (creates > packagePromo.getMaxCreates()) {
-                  int overage = Ints.saturatedCast(creates) - packagePromo.getMaxCreates();
-                  logger.atInfo().log(
-                      "Package with package token %s has exceeded their max domain creation limit"
-                          + " by %d name(s).",
-                      packagePromo.getToken().getKey(), overage);
-                  packagesOverCreateLimit.add(packagePromo);
-                }
-              }
-              if (packagesOverCreateLimit.build().isEmpty()) {
-                logger.atInfo().log("Found no packages over their create limit.");
-              } else {
-                logger.atInfo().log(
-                    "Found %d packages over their create limit.",
-                    packagesOverCreateLimit.build().size());
-                for (PackagePromotion packagePromotion : packagesOverCreateLimit.build()) {
-                  AllocationToken packageToken = tm().loadByKey(packagePromotion.getToken());
-                  Optional<Registrar> registrar =
-                      Registrar.loadByRegistrarIdCached(
-                          packageToken.getAllowedRegistrarIds().iterator().next());
-                  if (registrar.isPresent()) {
-                    String body =
-                        String.format(
-                            packageCreateLimitEmailBodyText,
-                            registrar.get().getRegistrarName(),
-                            packageToken.getToken(),
-                            registrySupportEmail);
-                    sendNotification(
-                        packageToken, packageCreateLimitEmailSubjectText, body, registrar.get());
-                  } else {
-                    logger.atSevere().log(
-                        String.format(
-                            "Could not find registrar for package token %s", packageToken));
-                  }
-                }
-              }
-            });
+    tm().transact(this::checkPackages);
+  }
+
+  private void checkPackages() {
+    ImmutableList<PackagePromotion> packages = tm().loadAllOf(PackagePromotion.class);
+    ImmutableMap.Builder<PackagePromotion, Long> packagesOverCreateLimitBuilder =
+        new ImmutableMap.Builder<>();
+    ImmutableMap.Builder<PackagePromotion, Long> packagesOverActiveDomainsLimitBuilder =
+        new ImmutableMap.Builder<>();
+    for (PackagePromotion packagePromo : packages) {
+      Long creates =
+          (Long)
+              tm().query(
+                      "SELECT COUNT(*) FROM DomainHistory WHERE current_package_token ="
+                          + " :token AND modificationTime >= :lastBilling AND type ="
+                          + " 'DOMAIN_CREATE'")
+                  .setParameter("token", packagePromo.getToken().getKey().toString())
+                  .setParameter("lastBilling", packagePromo.getNextBillingDate().minusYears(1))
+                  .getSingleResult();
+      if (creates > packagePromo.getMaxCreates()) {
+        long overage = creates - packagePromo.getMaxCreates();
+        logger.atInfo().log(
+            "Package with package token %s has exceeded their max domain creation limit"
+                + " by %d name(s).",
+            packagePromo.getToken().getKey(), overage);
+        packagesOverCreateLimitBuilder.put(packagePromo, creates);
+      }
+
+      Long activeDomains =
+          tm().query(
+                  "SELECT COUNT(*) FROM Domain WHERE currentPackageToken = :token"
+                      + " AND deletionTime = :endOfTime",
+                  Long.class)
+              .setParameter("token", packagePromo.getToken())
+              .setParameter("endOfTime", END_OF_TIME)
+              .getSingleResult();
+      if (activeDomains > packagePromo.getMaxDomains()) {
+        int overage = Ints.saturatedCast(activeDomains) - packagePromo.getMaxDomains();
+        logger.atInfo().log(
+            "Package with package token %s has exceed their max active domains limit by"
+                + " %d name(s).",
+            packagePromo.getToken().getKey(), overage);
+        packagesOverActiveDomainsLimitBuilder.put(packagePromo, activeDomains);
+      }
+    }
+    handlePackageCreationOverage(packagesOverCreateLimitBuilder.build());
+    handleActiveDomainOverage(packagesOverActiveDomainsLimitBuilder.build());
+  }
+
+  private void handlePackageCreationOverage(ImmutableMap<PackagePromotion, Long> overageList) {
+    if (overageList.isEmpty()) {
+      logger.atInfo().log("Found no packages over their create limit.");
+      return;
+    }
+    logger.atInfo().log("Found %d packages over their create limit.", overageList.size());
+    for (PackagePromotion packagePromotion : overageList.keySet()) {
+      AllocationToken packageToken = tm().loadByKey(packagePromotion.getToken());
+      Optional<Registrar> registrar =
+          Registrar.loadByRegistrarIdCached(
+              Iterables.getOnlyElement(packageToken.getAllowedRegistrarIds()));
+      if (registrar.isPresent()) {
+        String body =
+            String.format(
+                packageCreateLimitEmailBody,
+                packagePromotion.getId(),
+                packageToken.getToken(),
+                registrar.get().getRegistrarName(),
+                packagePromotion.getMaxCreates(),
+                overageList.get(packagePromotion));
+        sendNotification(packageToken, packageCreateLimitEmailSubject, body, registrar.get());
+      } else {
+        throw new IllegalStateException(
+            String.format("Could not find registrar for package token %s", packageToken));
+      }
+    }
+  }
+
+  private void handleActiveDomainOverage(ImmutableMap<PackagePromotion, Long> overageList) {
+    if (overageList.isEmpty()) {
+      logger.atInfo().log("Found no packages over their active domains limit.");
+      return;
+    }
+    logger.atInfo().log("Found %d packages over their active domains limit.", overageList.size());
+    for (PackagePromotion packagePromotion : overageList.keySet()) {
+      int daysSinceLastNotification =
+          packagePromotion
+              .getLastNotificationSent()
+              .map(sentDate -> Days.daysBetween(sentDate, clock.nowUtc()).getDays())
+              .orElse(Integer.MAX_VALUE);
+      if (daysSinceLastNotification < THIRTY_DAYS) {
+        // Don't send an email if notification was already sent within the last 30
+        // days
+        continue;
+      } else if (daysSinceLastNotification < FORTY_DAYS) {
+        // Send an upgrade email if last email was between 30 and 40 days ago
+        sendActiveDomainOverageEmail(
+            /* warning= */ false, packagePromotion, overageList.get(packagePromotion));
+      } else {
+        // Send a warning email
+        sendActiveDomainOverageEmail(
+            /* warning= */ true, packagePromotion, overageList.get(packagePromotion));
+      }
+    }
+  }
+
+  private void sendActiveDomainOverageEmail(
+      boolean warning, PackagePromotion packagePromotion, long activeDomains) {
+    String emailSubject =
+        warning ? packageDomainLimitWarningEmailSubject : packageDomainLimitUpgradeEmailSubject;
+    String emailTemplate =
+        warning ? packageDomainLimitWarningEmailBody : packageDomainLimitUpgradeEmailBody;
+    AllocationToken packageToken = tm().loadByKey(packagePromotion.getToken());
+    Optional<Registrar> registrar =
+        Registrar.loadByRegistrarIdCached(
+            Iterables.getOnlyElement(packageToken.getAllowedRegistrarIds()));
+    if (registrar.isPresent()) {
+      String body =
+          String.format(
+              emailTemplate,
+              packagePromotion.getId(),
+              packageToken.getToken(),
+              registrar.get().getRegistrarName(),
+              packagePromotion.getMaxDomains(),
+              activeDomains);
+      sendNotification(packageToken, emailSubject, body, registrar.get());
+      tm().put(packagePromotion.asBuilder().setLastNotificationSent(clock.nowUtc()).build());
+    } else {
+      throw new IllegalStateException(
+          String.format("Could not find registrar for package token %s", packageToken));
+    }
   }
 
   private void sendNotification(
       AllocationToken packageToken, String subject, String body, Registrar registrar) {
     logger.atInfo().log(
         String.format(
-            "Compliance email sent to the %s registrar regarding the package with token" + " %s.",
+            "Compliance email sent to support regarding the %s registrar and the package with token"
+                + " %s.",
             registrar.getRegistrarName(), packageToken.getToken()));
-    sendEmailUtils.sendEmail(
-        subject,
-        body,
-        Optional.of(registrySupportEmail),
-        registrar.getContacts().stream()
-            .filter(c -> c.getTypes().contains(RegistrarPoc.Type.ADMIN))
-            .map(RegistrarPoc::getEmailAddress)
-            .collect(toImmutableList()));
+    sendEmailUtils.sendEmail(subject, body, ImmutableList.of(registrySupportEmail));
   }
 }

core/src/main/java/google/registry/batch/CloudTasksUtils.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.batch;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableList.toImmutableList;
@@ -36,12 +36,18 @@ import com.google.common.net.MediaType;
 import com.google.common.net.UrlEscapers;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.util.Timestamps;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.request.Action.Service;
+import google.registry.util.Clock;
+import google.registry.util.CollectionUtils;
+import google.registry.util.Retrier;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Optional;
 import java.util.Random;
 import java.util.function.Supplier;
+import javax.inject.Inject;
 import org.joda.time.Duration;
 
 /** Utilities for dealing with Cloud Tasks. */
@@ -57,11 +63,12 @@ public class CloudTasksUtils implements Serializable {
   private final String locationId;
   private final SerializableCloudTasksClient client;
 
+  @Inject
   public CloudTasksUtils(
       Retrier retrier,
       Clock clock,
-      String projectId,
-      String locationId,
+      @Config("projectId") String projectId,
+      @Config("locationId") String locationId,
       SerializableCloudTasksClient client) {
     this.retrier = retrier;
     this.clock = clock;
@@ -108,7 +115,7 @@ public class CloudTasksUtils implements Serializable {
    *     the worker service</a>
    */
   private Task createTask(
-      String path, HttpMethod method, String service, Multimap<String, String> params) {
+      String path, HttpMethod method, Service service, Multimap<String, String> params) {
     checkArgument(
         path != null && !path.isEmpty() && path.charAt(0) == '/',
         "The path must start with a '/'.");
@@ -119,7 +126,8 @@ public class CloudTasksUtils implements Serializable {
     AppEngineHttpRequest.Builder requestBuilder =
         AppEngineHttpRequest.newBuilder()
             .setHttpMethod(method)
-            .setAppEngineRouting(AppEngineRouting.newBuilder().setService(service).build());
+            .setAppEngineRouting(
+                AppEngineRouting.newBuilder().setService(service.toString()).build());
 
     if (!CollectionUtils.isNullOrEmpty(params)) {
       Escaper escaper = UrlEscapers.urlPathSegmentEscaper();
@@ -165,7 +173,7 @@ public class CloudTasksUtils implements Serializable {
   private Task createTaskWithJitter(
       String path,
       HttpMethod method,
-      String service,
+      Service service,
       Multimap<String, String> params,
       Optional<Integer> jitterSeconds) {
     if (!jitterSeconds.isPresent() || jitterSeconds.get() <= 0) {
@@ -199,7 +207,7 @@ public class CloudTasksUtils implements Serializable {
   private Task createTaskWithDelay(
       String path,
       HttpMethod method,
-      String service,
+      Service service,
       Multimap<String, String> params,
       Duration delay) {
     if (delay.isEqual(Duration.ZERO)) {
@@ -211,11 +219,11 @@ public class CloudTasksUtils implements Serializable {
         .build();
   }
 
-  public Task createPostTask(String path, String service, Multimap<String, String> params) {
+  public Task createPostTask(String path, Service service, Multimap<String, String> params) {
     return createTask(path, HttpMethod.POST, service, params);
   }
 
-  public Task createGetTask(String path, String service, Multimap<String, String> params) {
+  public Task createGetTask(String path, Service service, Multimap<String, String> params) {
     return createTask(path, HttpMethod.GET, service, params);
   }
 
@@ -224,7 +232,7 @@ public class CloudTasksUtils implements Serializable {
    */
   public Task createPostTaskWithJitter(
       String path,
-      String service,
+      Service service,
       Multimap<String, String> params,
       Optional<Integer> jitterSeconds) {
     return createTaskWithJitter(path, HttpMethod.POST, service, params, jitterSeconds);
@@ -235,7 +243,7 @@ public class CloudTasksUtils implements Serializable {
    */
   public Task createGetTaskWithJitter(
       String path,
-      String service,
+      Service service,
       Multimap<String, String> params,
       Optional<Integer> jitterSeconds) {
     return createTaskWithJitter(path, HttpMethod.GET, service, params, jitterSeconds);
@@ -243,13 +251,13 @@ public class CloudTasksUtils implements Serializable {
 
   /** Create a {@link Task} via HTTP.POST that will be delayed for {@code delay}. */
   public Task createPostTaskWithDelay(
-      String path, String service, Multimap<String, String> params, Duration delay) {
+      String path, Service service, Multimap<String, String> params, Duration delay) {
     return createTaskWithDelay(path, HttpMethod.POST, service, params, delay);
   }
 
   /** Create a {@link Task} via HTTP.GET that will be delayed for {@code delay}. */
   public Task createGetTaskWithDelay(
-      String path, String service, Multimap<String, String> params, Duration delay) {
+      String path, Service service, Multimap<String, String> params, Duration delay) {
     return createTaskWithDelay(path, HttpMethod.GET, service, params, delay);
   }
 

core/src/main/java/google/registry/batch/DeleteProberDataAction.java

@@ -19,9 +19,9 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.batch.BatchModule.PARAM_DRY_RUN;
 import static google.registry.config.RegistryEnvironment.PRODUCTION;
+import static google.registry.dns.DnsUtils.requestDomainDnsRefresh;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
 import static google.registry.model.tld.Registries.getTldsOfType;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
@@ -33,12 +33,11 @@ import com.google.common.collect.Sets;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
-import google.registry.dns.DnsQueue;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.EppResourceUtils;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainHistory;
-import google.registry.model.tld.Registry.TldType;
+import google.registry.model.tld.Tld.TldType;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
@@ -99,8 +98,6 @@ public class DeleteProberDataAction implements Runnable {
   /** Number of domains to retrieve and delete per SQL transaction. */
   private static final int BATCH_SIZE = 1000;
 
-  @Inject DnsQueue dnsQueue;
-
   @Inject
   @Parameter(PARAM_DRY_RUN)
   boolean isDryRun;
@@ -140,7 +137,7 @@ public class DeleteProberDataAction implements Runnable {
   private void runSqlJob(ImmutableSet<String> deletableTlds) {
     AtomicInteger softDeletedDomains = new AtomicInteger();
     AtomicInteger hardDeletedDomains = new AtomicInteger();
-    jpaTm().transact(() -> processDomains(deletableTlds, softDeletedDomains, hardDeletedDomains));
+    tm().transact(() -> processDomains(deletableTlds, softDeletedDomains, hardDeletedDomains));
     logger.atInfo().log(
         "%s %d domains.",
         isDryRun ? "Would have soft-deleted" : "Soft-deleted", softDeletedDomains.get());
@@ -157,8 +154,7 @@ public class DeleteProberDataAction implements Runnable {
     // Scroll through domains, soft-deleting as necessary (very few will be soft-deleted) and
     // keeping track of which domains to hard-delete (there can be many, so we batch them up)
     try (ScrollableResults scrollableResult =
-        jpaTm()
-            .query(DOMAIN_QUERY_STRING, Domain.class)
+        tm().query(DOMAIN_QUERY_STRING, Domain.class)
             .setParameter("tlds", deletableTlds)
             .setParameter(
                 "creationTimeCutoff", CreateAutoTimestamp.create(now.minus(DOMAIN_USED_DURATION)))
@@ -183,8 +179,8 @@ public class DeleteProberDataAction implements Runnable {
               domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
           domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
           hostNamesToHardDelete = new ImmutableList.Builder<>();
-          jpaTm().getEntityManager().flush();
-          jpaTm().getEntityManager().clear();
+          tm().getEntityManager().flush();
+          tm().getEntityManager().clear();
         }
       }
       // process the remainder
@@ -224,34 +220,27 @@ public class DeleteProberDataAction implements Runnable {
     }
   }
 
-  private void hardDeleteDomainsAndHosts(
+  private static void hardDeleteDomainsAndHosts(
       ImmutableList<String> domainRepoIds, ImmutableList<String> hostNames) {
-    jpaTm()
-        .query("DELETE FROM Host WHERE hostName IN :hostNames")
+    tm().query("DELETE FROM Host WHERE hostName IN :hostNames")
         .setParameter("hostNames", hostNames)
         .executeUpdate();
-    jpaTm()
-        .query("DELETE FROM BillingEvent WHERE domainRepoId IN :repoIds")
+    tm().query("DELETE FROM BillingEvent WHERE domainRepoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
-    jpaTm()
-        .query("DELETE FROM BillingRecurrence WHERE domainRepoId IN :repoIds")
+    tm().query("DELETE FROM BillingRecurrence WHERE domainRepoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
-    jpaTm()
-        .query("DELETE FROM BillingCancellation WHERE domainRepoId IN :repoIds")
+    tm().query("DELETE FROM BillingCancellation WHERE domainRepoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
-    jpaTm()
-        .query("DELETE FROM DomainHistory WHERE repoId IN :repoIds")
+    tm().query("DELETE FROM DomainHistory WHERE repoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
-    jpaTm()
-        .query("DELETE FROM PollMessage WHERE domainRepoId IN :repoIds")
+    tm().query("DELETE FROM PollMessage WHERE domainRepoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
-    jpaTm()
-        .query("DELETE FROM Domain WHERE repoId IN :repoIds")
+    tm().query("DELETE FROM Domain WHERE repoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
   }
@@ -273,6 +262,6 @@ public class DeleteProberDataAction implements Runnable {
     // messages, or auto-renews because those will all be hard-deleted the next time the job runs
     // anyway.
     tm().putAll(ImmutableList.of(deletedDomain, historyEntry));
-    dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
+    requestDomainDnsRefresh(deletedDomain.getDomainName());
   }
 }

core/src/main/java/google/registry/batch/ExpandBillingRecurrencesAction.java

@@ -0,0 +1,163 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.batch.BatchModule.PARAM_DRY_RUN;
+import static google.registry.beam.BeamUtils.createJobName;
+import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.billing.ExpandBillingRecurrencesPipeline;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingRecurrence;
+import google.registry.model.common.Cursor;
+import google.registry.request.Action;
+import google.registry.request.Parameter;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import java.io.IOException;
+import java.util.Optional;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+
+/**
+ * An action that kicks off a {@link ExpandBillingRecurrencesPipeline} dataflow job to expand {@link
+ * BillingRecurrence} billing events into synthetic {@link BillingEvent} events.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = "/_dr/task/expandBillingRecurrences",
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class ExpandBillingRecurrencesAction implements Runnable {
+
+  public static final String PARAM_START_TIME = "startTime";
+  public static final String PARAM_END_TIME = "endTime";
+  public static final String PARAM_ADVANCE_CURSOR = "advanceCursor";
+
+  private static final String PIPELINE_NAME = "expand_billing_recurrences_pipeline";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  @Inject Clock clock;
+
+  @Inject
+  @Parameter(PARAM_DRY_RUN)
+  boolean isDryRun;
+
+  @Inject
+  @Parameter(PARAM_ADVANCE_CURSOR)
+  boolean advanceCursor;
+
+  @Inject
+  @Parameter(PARAM_START_TIME)
+  Optional<DateTime> startTimeParam;
+
+  @Inject
+  @Parameter(PARAM_END_TIME)
+  Optional<DateTime> endTimeParam;
+
+  @Inject
+  @Config("projectId")
+  String projectId;
+
+  @Inject
+  @Config("defaultJobRegion")
+  String jobRegion;
+
+  @Inject
+  @Config("beamStagingBucketUrl")
+  String stagingBucketUrl;
+
+  @Inject Dataflow dataflow;
+
+  @Inject Response response;
+
+  @Inject
+  ExpandBillingRecurrencesAction() {}
+
+  @Override
+  public void run() {
+    checkArgument(!(isDryRun && advanceCursor), "Cannot advance the cursor in a dry run.");
+    DateTime endTime = endTimeParam.orElse(clock.nowUtc());
+    checkArgument(
+        !endTime.isAfter(clock.nowUtc()), "End time (%s) must be at or before now", endTime);
+    DateTime startTime =
+        startTimeParam.orElse(
+            tm().transact(
+                    () ->
+                        tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
+                            .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
+                            .getCursorTime()));
+    checkArgument(
+        startTime.isBefore(endTime),
+        String.format("Start time (%s) must be before end time (%s)", startTime, endTime));
+    LaunchFlexTemplateParameter launchParameter =
+        new LaunchFlexTemplateParameter()
+            .setJobName(
+                createJobName(
+                    String.format(
+                        "expand-billing-%s", startTime.toString("yyyy-MM-dd't'HH-mm-ss'z'")),
+                    clock))
+            .setContainerSpecGcsPath(
+                String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+            .setParameters(
+                new ImmutableMap.Builder<String, String>()
+                    .put("registryEnvironment", RegistryEnvironment.get().name())
+                    .put("startTime", startTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"))
+                    .put("endTime", endTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"))
+                    .put("isDryRun", Boolean.toString(isDryRun))
+                    .put("advanceCursor", Boolean.toString(advanceCursor))
+                    .build());
+    logger.atInfo().log(
+        "Launching billing recurrence expansion pipeline for event time range [%s, %s)%s.",
+        startTime,
+        endTime,
+        isDryRun ? " in dry run mode" : advanceCursor ? "" : " without advancing the cursor");
+    try {
+      LaunchFlexTemplateResponse launchResponse =
+          dataflow
+              .projects()
+              .locations()
+              .flexTemplates()
+              .launch(
+                  projectId,
+                  jobRegion,
+                  new LaunchFlexTemplateRequest().setLaunchParameter(launchParameter))
+              .execute();
+      logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
+      response.setStatus(SC_OK);
+      response.setPayload(
+          String.format(
+              "Launched billing recurrence expansion pipeline: %s",
+              launchResponse.getJob().getId()));
+    } catch (IOException e) {
+      logger.atWarning().withCause(e).log("Pipeline Launch failed");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
+    }
+  }
+}

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -1,379 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.batch;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.collect.Sets.difference;
-import static com.google.common.collect.Sets.newHashSet;
-import static google.registry.batch.BatchModule.PARAM_DRY_RUN;
-import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
-import static google.registry.model.domain.Period.Unit.YEARS;
-import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
-import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.util.CollectionUtils.union;
-import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static google.registry.util.DateTimeUtils.earliestOf;
-import static google.registry.util.DomainNameUtils.getTldFromDomainName;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Range;
-import com.google.common.collect.Streams;
-import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.flows.domain.DomainPricingLogic;
-import google.registry.model.ImmutableObject;
-import google.registry.model.billing.BillingEvent;
-import google.registry.model.billing.BillingEvent.Flag;
-import google.registry.model.billing.BillingEvent.OneTime;
-import google.registry.model.billing.BillingEvent.Recurring;
-import google.registry.model.common.Cursor;
-import google.registry.model.domain.Domain;
-import google.registry.model.domain.DomainHistory;
-import google.registry.model.domain.Period;
-import google.registry.model.reporting.DomainTransactionRecord;
-import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
-import google.registry.model.tld.Registry;
-import google.registry.persistence.VKey;
-import google.registry.request.Action;
-import google.registry.request.Parameter;
-import google.registry.request.Response;
-import google.registry.request.auth.Auth;
-import google.registry.util.Clock;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import javax.inject.Inject;
-import org.joda.time.DateTime;
-
-/**
- * An action that expands {@link Recurring} billing events into synthetic {@link OneTime} events.
- *
- * <p>The cursor used throughout this action (overridden if necessary using the parameter {@code
- * cursorTime}) represents the inclusive lower bound on the range of billing times that will be
- * expanded as a result of the job (the exclusive upper bound being the execution time of the job).
- */
-@Action(
-    service = Action.Service.BACKEND,
-    path = "/_dr/task/expandRecurringBillingEvents",
-    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
-public class ExpandRecurringBillingEventsAction implements Runnable {
-
-  public static final String PARAM_CURSOR_TIME = "cursorTime";
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
-  @Inject Clock clock;
-
-  @Inject
-  @Config("jdbcBatchSize")
-  int batchSize;
-
-  @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
-  @Inject @Parameter(PARAM_CURSOR_TIME) Optional<DateTime> cursorTimeParam;
-
-  @Inject DomainPricingLogic domainPricingLogic;
-  @Inject Response response;
-  @Inject ExpandRecurringBillingEventsAction() {}
-
-  @Override
-  public void run() {
-    DateTime executeTime = clock.nowUtc();
-    DateTime persistedCursorTime =
-        tm().transact(
-                () ->
-                    tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
-                        .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
-                        .getCursorTime());
-    DateTime cursorTime = cursorTimeParam.orElse(persistedCursorTime);
-    checkArgument(
-        cursorTime.isBefore(executeTime), "Cursor time must be earlier than execution time.");
-    logger.atInfo().log(
-        "Running Recurring billing event expansion for billing time range [%s, %s).",
-        cursorTime, executeTime);
-    expandSqlBillingEventsInBatches(executeTime, cursorTime, persistedCursorTime);
-  }
-
-  private void expandSqlBillingEventsInBatches(
-      DateTime executeTime, DateTime cursorTime, DateTime persistedCursorTime) {
-    int totalBillingEventsSaved = 0;
-    long maxProcessedRecurrenceId = 0;
-    SqlBatchResults sqlBatchResults;
-
-    do {
-      final long prevMaxProcessedRecurrenceId = maxProcessedRecurrenceId;
-      sqlBatchResults =
-          jpaTm()
-              .transact(
-                  () -> {
-                    Set<String> expandedDomains = newHashSet();
-                    int batchBillingEventsSaved = 0;
-                    long maxRecurrenceId = prevMaxProcessedRecurrenceId;
-                    List<Recurring> recurrings =
-                        jpaTm()
-                            .query(
-                                "FROM BillingRecurrence "
-                                    + "WHERE eventTime <= :executeTime "
-                                    + "AND eventTime < recurrenceEndTime "
-                                    + "AND id > :maxProcessedRecurrenceId "
-                                    + "AND recurrenceEndTime > :adjustedCursorTime "
-                                    + "ORDER BY id ASC",
-                                Recurring.class)
-                            .setParameter("executeTime", executeTime)
-                            .setParameter("maxProcessedRecurrenceId", prevMaxProcessedRecurrenceId)
-                            .setParameter(
-                                "adjustedCursorTime",
-                                cursorTime.minus(Registry.DEFAULT_AUTO_RENEW_GRACE_PERIOD))
-                            .setMaxResults(batchSize)
-                            .getResultList();
-                    for (Recurring recurring : recurrings) {
-                      if (expandedDomains.contains(recurring.getTargetId())) {
-                        // On the off chance this batch contains multiple recurrences for the same
-                        // domain (which is actually possible if a given domain is quickly renewed
-                        // multiple times in a row), then short-circuit after the first one is
-                        // processed that involves actually expanding a billing event. This is
-                        // necessary because otherwise we get an "Inserted/updated object reloaded"
-                        // error from Hibernate when those billing events would be loaded
-                        // inside a transaction where they were already written. Note, there is no
-                        // actual further work to be done in this case anyway, not unless it has
-                        // somehow been over a year since this action last ran successfully (and if
-                        // that were somehow true, the remaining billing events would still be
-                        // expanded on subsequent runs).
-                        continue;
-                      }
-                      int billingEventsSaved =
-                          expandBillingEvent(
-                              recurring, executeTime, cursorTime, isDryRun, domainPricingLogic);
-                      batchBillingEventsSaved += billingEventsSaved;
-                      if (billingEventsSaved > 0) {
-                        expandedDomains.add(recurring.getTargetId());
-                      }
-                      maxRecurrenceId = Math.max(maxRecurrenceId, recurring.getId());
-                    }
-                    return SqlBatchResults.create(
-                        batchBillingEventsSaved,
-                        maxRecurrenceId,
-                        maxRecurrenceId > prevMaxProcessedRecurrenceId);
-                  });
-      totalBillingEventsSaved += sqlBatchResults.batchBillingEventsSaved();
-      maxProcessedRecurrenceId = sqlBatchResults.maxProcessedRecurrenceId();
-      if (sqlBatchResults.batchBillingEventsSaved() > 0) {
-        logger.atInfo().log(
-            "Saved %d billing events in batch (%d total) with max recurrence id %d.",
-            sqlBatchResults.batchBillingEventsSaved(),
-            totalBillingEventsSaved,
-            maxProcessedRecurrenceId);
-      } else {
-        // If we're churning through a lot of no-op recurrences that don't need expanding (yet?),
-        // then only log one no-op every so often as a good balance between letting the user track
-        // that the action is still running while also not spamming the logs incessantly.
-        logger.atInfo().atMostEvery(3, TimeUnit.MINUTES).log(
-            "Processed up to max recurrence id %d (no billing events saved recently).",
-            maxProcessedRecurrenceId);
-      }
-    } while (sqlBatchResults.shouldContinue());
-
-    if (!isDryRun) {
-      logger.atInfo().log("Saved %d total OneTime billing events.", totalBillingEventsSaved);
-    } else {
-      logger.atInfo().log(
-          "Generated %d total OneTime billing events (dry run).", totalBillingEventsSaved);
-    }
-    logger.atInfo().log(
-        "Recurring event expansion %s complete for billing event range [%s, %s).",
-        isDryRun ? "(dry run) " : "", cursorTime, executeTime);
-    tm().transact(
-            () -> {
-              // Check for the unlikely scenario where the cursor has been altered during the
-              // expansion.
-              DateTime currentCursorTime =
-                  tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
-                      .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
-                      .getCursorTime();
-              if (!currentCursorTime.equals(persistedCursorTime)) {
-                throw new IllegalStateException(
-                    String.format(
-                        "Current cursor position %s does not match persisted cursor position %s.",
-                        currentCursorTime, persistedCursorTime));
-              }
-              if (!isDryRun) {
-                tm().put(Cursor.createGlobal(RECURRING_BILLING, executeTime));
-              }
-            });
-  }
-
-  @AutoValue
-  abstract static class SqlBatchResults {
-    abstract int batchBillingEventsSaved();
-
-    abstract long maxProcessedRecurrenceId();
-
-    abstract boolean shouldContinue();
-
-    static SqlBatchResults create(
-        int batchBillingEventsSaved, long maxProcessedRecurrenceId, boolean shouldContinue) {
-      return new AutoValue_ExpandRecurringBillingEventsAction_SqlBatchResults(
-          batchBillingEventsSaved, maxProcessedRecurrenceId, shouldContinue);
-    }
-  }
-
-  private static int expandBillingEvent(
-      Recurring recurring,
-      DateTime executeTime,
-      DateTime cursorTime,
-      boolean isDryRun,
-      DomainPricingLogic domainPricingLogic) {
-    ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder = new ImmutableSet.Builder<>();
-    final Registry tld = Registry.get(getTldFromDomainName(recurring.getTargetId()));
-
-    // Determine the complete set of times at which this recurring event should
-    // occur (up to and including the runtime of the action).
-    Iterable<DateTime> eventTimes =
-        recurring
-            .getRecurrenceTimeOfYear()
-            .getInstancesInRange(
-                Range.closed(
-                    recurring.getEventTime(),
-                    earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
-
-    // Convert these event times to billing times
-    final ImmutableSet<DateTime> billingTimes =
-        getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
-
-    VKey<Domain> domainKey = VKey.create(Domain.class, recurring.getDomainRepoId());
-    Iterable<OneTime> oneTimesForDomain;
-    oneTimesForDomain =
-        tm().createQueryComposer(OneTime.class)
-            .where("domainRepoId", EQ, recurring.getDomainRepoId())
-            .list();
-
-    // Determine the billing times that already have OneTime events persisted.
-    ImmutableSet<DateTime> existingBillingTimes =
-        getExistingBillingTimes(oneTimesForDomain, recurring);
-
-    ImmutableSet.Builder<DomainHistory> historyEntriesBuilder = new ImmutableSet.Builder<>();
-    // Create synthetic OneTime events for all billing times that do not yet have
-    // an event persisted.
-    for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
-      // Construct a new HistoryEntry that parents over the OneTime
-      DomainHistory historyEntry =
-          new DomainHistory.Builder()
-              .setBySuperuser(false)
-              .setRegistrarId(recurring.getRegistrarId())
-              .setModificationTime(tm().getTransactionTime())
-              .setDomain(tm().loadByKey(domainKey))
-              .setPeriod(Period.create(1, YEARS))
-              .setReason("Domain autorenewal by ExpandRecurringBillingEventsAction")
-              .setRequestedByRegistrar(false)
-              .setType(DOMAIN_AUTORENEW)
-              // Note: the following statement seems to not be entirely correct as manual renewal
-              // during the autorenew grace period also closes out the existing recurrence, but in
-              // that instance the autorenew history entry should still have the transaction records
-              // for obvious reasons. It can be argued the history entry should always have the
-              // transaction record, regardless of what happens afterward. If the domain is deleted
-              // later during the autorenew grace period, another history entry for the delete would
-              // record that mutation separately, but the previous autorenew should not have its
-              // history entry retroactively altered, or in this case have the transaction records
-              // omitted when its created belatedly (when billing time is in scope). However, since
-              // we will be rewriting this action and only want to do the absolute minimum change to
-              // fix it for now, we will leave the current logic in place to avoid any unnecessary
-              // complications.
-              //
-              // Don't write a domain transaction record if the recurrence was
-              // ended prior to the billing time (i.e. a domain was deleted
-              // during the autorenew grace period).
-              .setDomainTransactionRecords(
-                  recurring.getRecurrenceEndTime().isBefore(billingTime)
-                      ? ImmutableSet.of()
-                      : ImmutableSet.of(
-                          DomainTransactionRecord.create(
-                              tld.getTldStr(),
-                              // We report this when the autorenew grace period
-                              // ends
-                              billingTime,
-                              TransactionReportField.netRenewsFieldFromYears(1),
-                              1)))
-              .build();
-      historyEntriesBuilder.add(historyEntry);
-
-      DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
-
-      syntheticOneTimesBuilder.add(
-          new OneTime.Builder()
-              .setBillingTime(billingTime)
-              .setRegistrarId(recurring.getRegistrarId())
-              // Determine the cost for a one-year renewal.
-              .setCost(
-                  domainPricingLogic
-                      .getRenewPrice(tld, recurring.getTargetId(), eventTime, 1, recurring)
-                      .getRenewCost())
-              .setEventTime(eventTime)
-              .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
-              .setDomainHistory(historyEntry)
-              .setPeriodYears(1)
-              .setReason(recurring.getReason())
-              .setSyntheticCreationTime(executeTime)
-              .setCancellationMatchingBillingEvent(recurring)
-              .setTargetId(recurring.getTargetId())
-              .build());
-    }
-    Set<DomainHistory> historyEntries = historyEntriesBuilder.build();
-    Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
-    if (!isDryRun) {
-      ImmutableSet<ImmutableObject> entitiesToSave =
-          new ImmutableSet.Builder<ImmutableObject>()
-              .addAll(historyEntries)
-              .addAll(syntheticOneTimes)
-              .build();
-      tm().putAll(entitiesToSave);
-    }
-    return syntheticOneTimes.size();
-  }
-
-  /**
-   * Filters a set of {@link DateTime}s down to event times that are in scope for a particular
-   * action run, given the cursor time and the action execution time.
-   */
-  protected static ImmutableSet<DateTime> getBillingTimesInScope(
-      Iterable<DateTime> eventTimes,
-      DateTime cursorTime,
-      DateTime executeTime,
-      final Registry tld) {
-    return Streams.stream(eventTimes)
-        .map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
-        .filter(Range.closedOpen(cursorTime, executeTime))
-        .collect(toImmutableSet());
-  }
-
-  /**
-   * Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted for a
-   * given recurring billing event.
-   */
-  private static ImmutableSet<DateTime> getExistingBillingTimes(
-      Iterable<BillingEvent.OneTime> oneTimesForDomain,
-      final BillingEvent.Recurring recurringEvent) {
-    return Streams.stream(oneTimesForDomain)
-        .filter(
-            billingEvent ->
-                recurringEvent
-                    .createVKey()
-                    .equals(billingEvent.getCancellationMatchingBillingEvent()))
-        .map(OneTime::getBillingTime)
-        .collect(toImmutableSet());
-  }
-}

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -16,7 +16,6 @@ package google.registry.batch;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
@@ -189,7 +188,7 @@ public class RelockDomainAction implements Runnable {
         "Domain %s has a pending delete.",
         domainName);
     checkArgument(
-        !DateTimeUtils.isAtOrAfter(jpaTm().getTransactionTime(), domain.getDeletionTime()),
+        !DateTimeUtils.isAtOrAfter(tm().getTransactionTime(), domain.getDeletionTime()),
         "Domain %s has been deleted.",
         domainName);
     checkArgument(

core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java

@@ -14,31 +14,39 @@
 
 package google.registry.batch;
 
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
-import static org.apache.http.HttpStatus.SC_OK;
+import static google.registry.batch.BatchModule.PARAM_DRY_RUN;
+import static google.registry.beam.BeamUtils.createJobName;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
+import google.registry.beam.wipeout.WipeOutContactHistoryPiiPipeline;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
 import google.registry.model.contact.ContactHistory;
 import google.registry.request.Action;
 import google.registry.request.Action.Service;
+import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Stream;
+import java.io.IOException;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
 /**
- * An action that wipes out Personal Identifiable Information (PII) fields of {@link ContactHistory}
- * entities.
+ * An action that launches {@link WipeOutContactHistoryPiiPipeline} to wipe out Personal
+ * Identifiable Information (PII) fields of {@link ContactHistory} entities.
  *
- * <p>ContactHistory entities should be retained in the database for only certain amount of time.
- * This periodic wipe out action only applies to SQL.
+ * <p>{@link ContactHistory} entities should be retained in the database for only certain amount of
+ * time.
  */
 @Action(
     service = Service.BACKEND,
@@ -47,92 +55,89 @@ import org.joda.time.DateTime;
 public class WipeOutContactHistoryPiiAction implements Runnable {
 
   public static final String PATH = "/_dr/task/wipeOutContactHistoryPii";
+  public static final String PARAM_CUTOFF_TIME = "wipeoutTime";
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final String PIPELINE_NAME = "wipe_out_contact_history_pii_pipeline";
+
   private final Clock clock;
-  private final Response response;
+  private final boolean isDryRun;
+  private final Optional<DateTime> maybeCutoffTime;
   private final int minMonthsBeforeWipeOut;
-  private final int wipeOutQueryBatchSize;
+  private final String stagingBucketUrl;
+  private final String projectId;
+  private final String jobRegion;
+  private final Dataflow dataflow;
+  private final Response response;
 
   @Inject
   public WipeOutContactHistoryPiiAction(
       Clock clock,
+      @Parameter(PARAM_DRY_RUN) boolean isDryRun,
+      @Parameter(PARAM_CUTOFF_TIME) Optional<DateTime> maybeCutoffTime,
       @Config("minMonthsBeforeWipeOut") int minMonthsBeforeWipeOut,
-      @Config("wipeOutQueryBatchSize") int wipeOutQueryBatchSize,
+      @Config("beamStagingBucketUrl") String stagingBucketUrl,
+      @Config("projectId") String projectId,
+      @Config("defaultJobRegion") String jobRegion,
+      Dataflow dataflow,
       Response response) {
     this.clock = clock;
-    this.response = response;
+    this.isDryRun = isDryRun;
+    this.maybeCutoffTime = maybeCutoffTime;
     this.minMonthsBeforeWipeOut = minMonthsBeforeWipeOut;
-    this.wipeOutQueryBatchSize = wipeOutQueryBatchSize;
+    this.stagingBucketUrl = stagingBucketUrl;
+    this.projectId = projectId;
+    this.jobRegion = jobRegion;
+    this.dataflow = dataflow;
+    this.response = response;
   }
 
   @Override
   public void run() {
     response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    DateTime cutoffTime =
+        maybeCutoffTime.orElse(clock.nowUtc().minusMonths(minMonthsBeforeWipeOut));
+    LaunchFlexTemplateParameter launchParameter =
+        new LaunchFlexTemplateParameter()
+            .setJobName(
+                createJobName(
+                    String.format(
+                        "contact-history-pii-wipeout-%s",
+                        cutoffTime.toString("yyyy-MM-dd't'HH-mm-ss'z'")),
+                    clock))
+            .setContainerSpecGcsPath(
+                String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+            .setParameters(
+                ImmutableMap.of(
+                    "registryEnvironment",
+                    RegistryEnvironment.get().name(),
+                    "cutoffTime",
+                    cutoffTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"),
+                    "isDryRun",
+                    Boolean.toString(isDryRun)));
+    logger.atInfo().log(
+        "Launching Beam pipeline to wipe out all PII of contact history entities prior to %s%s.",
+        cutoffTime, " in dry run mode");
     try {
-      int totalNumOfWipedEntities = 0;
-      DateTime wipeOutTime = clock.nowUtc().minusMonths(minMonthsBeforeWipeOut);
-      logger.atInfo().log(
-          "About to wipe out all PII of contact history entities prior to %s.", wipeOutTime);
-
-      int numOfWipedEntities = 0;
-      do {
-        numOfWipedEntities =
-            jpaTm()
-                .transact(
-                    () ->
-                        wipeOutContactHistoryData(
-                            getNextContactHistoryEntitiesWithPiiBatch(wipeOutTime)));
-        totalNumOfWipedEntities += numOfWipedEntities;
-      } while (numOfWipedEntities > 0);
-      String msg =
-          String.format(
-              "Done. Wiped out PII of %d ContactHistory entities in total.",
-              totalNumOfWipedEntities);
-      logger.atInfo().log(msg);
-      response.setPayload(msg);
+      LaunchFlexTemplateResponse launchResponse =
+          dataflow
+              .projects()
+              .locations()
+              .flexTemplates()
+              .launch(
+                  projectId,
+                  jobRegion,
+                  new LaunchFlexTemplateRequest().setLaunchParameter(launchParameter))
+              .execute();
+      logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
       response.setStatus(SC_OK);
-
-    } catch (Exception e) {
-      logger.atSevere().withCause(e).log(
-          "Exception thrown during the process of wiping out contact history PII.");
-      response.setStatus(SC_INTERNAL_SERVER_ERROR);
       response.setPayload(
           String.format(
-              "Exception thrown during the process of wiping out contact history PII with cause"
-                  + ": %s",
-              e));
+              "Launched contact history PII wipeout pipeline: %s",
+              launchResponse.getJob().getId()));
+    } catch (IOException e) {
+      logger.atWarning().withCause(e).log("Pipeline Launch failed");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
     }
   }
-
-  /**
-   * Returns a stream of up to {@link #wipeOutQueryBatchSize} {@link ContactHistory} entities
-   * containing PII that are prior to @param wipeOutTime.
-   */
-  @VisibleForTesting
-  Stream<ContactHistory> getNextContactHistoryEntitiesWithPiiBatch(DateTime wipeOutTime) {
-    // email is one of the required fields in EPP, meaning it's initially not null.
-    // Therefore, checking if it's null is one way to avoid processing contact history entities
-    // that have been processed previously. Refer to RFC 5733 for more information.
-    return jpaTm()
-        .query(
-            "FROM ContactHistory WHERE modificationTime < :wipeOutTime " + "AND email IS NOT NULL",
-            ContactHistory.class)
-        .setParameter("wipeOutTime", wipeOutTime)
-        .setMaxResults(wipeOutQueryBatchSize)
-        .getResultStream();
-  }
-
-  /** Wipes out the PII of each of the {@link ContactHistory} entities in the stream. */
-  @VisibleForTesting
-  int wipeOutContactHistoryData(Stream<ContactHistory> contactHistoryEntities) {
-    AtomicInteger numOfEntities = new AtomicInteger(0);
-    contactHistoryEntities.forEach(
-        contactHistoryEntity -> {
-          jpaTm().update(contactHistoryEntity.asBuilder().wipeOutPii().build());
-          numOfEntities.incrementAndGet();
-        });
-    logger.atInfo().log(
-        "Wiped out all PII fields of %d ContactHistory entities.", numOfEntities.get());
-    return numOfEntities.get();
-  }
 }

core/src/main/java/google/registry/batch/cannedscript/CannedScripts.java

@@ -0,0 +1,199 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch.cannedscript;
+
+import com.google.api.gax.core.FixedCredentialsProvider;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dns.Dns;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.tasks.v2.CloudTasksClient;
+import com.google.cloud.tasks.v2.CloudTasksSettings;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.flogger.FluentLogger;
+import dagger.Component;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.CredentialModule;
+import google.registry.config.CredentialModule.ApplicationDefaultCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.util.GoogleCredentialsBundle;
+import google.registry.util.UtilsModule;
+import java.io.IOException;
+import java.util.Optional;
+import javax.inject.Singleton;
+
+/** Canned actions invoked from {@link google.registry.batch.CannedScriptExecutionAction}. */
+// TODO(b/277239043): remove class after credential changes are rolled out.
+public class CannedScripts {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static final Supplier<CannedScriptsComponent> COMPONENT_SUPPLIER =
+      Suppliers.memoize(DaggerCannedScripts_CannedScriptsComponent::create);
+
+  public static void runAllChecks() {
+    CannedScriptsComponent component = COMPONENT_SUPPLIER.get();
+    String projectId = component.projectId();
+    Bigquery bigquery = component.bigQuery();
+    try {
+      bigquery.datasets().list(projectId).execute().getDatasets().stream()
+          .findAny()
+          .ifPresent(
+              datasets ->
+                  logger.atInfo().log("Found a BQ dataset [%s]", datasets.getFriendlyName()));
+      logger.atInfo().log("Finished accessing BQ.");
+    } catch (IOException ioe) {
+      logger.atSevere().withCause(ioe).log("Failed to access bigquery.");
+    }
+    try {
+      Dataflow dataflow = component.dataflow();
+      dataflow.projects().jobs().list(projectId).execute().getJobs().stream()
+          .findAny()
+          .ifPresent(job -> logger.atInfo().log("Found a job [%s]", job.getName()));
+      logger.atInfo().log("Finished accessing Dataflow.");
+    } catch (IOException ioe) {
+      logger.atSevere().withCause(ioe).log("Failed to access dataflow.");
+    }
+    try {
+      Storage gcs = component.gcs();
+      gcs.listAcls(projectId + "-beam");
+      logger.atInfo().log("Finished accessing gcs.");
+    } catch (RuntimeException e) {
+      logger.atSevere().withCause(e).log("Failed to access gcs.");
+    }
+    try {
+      Dns dns = component.dns();
+      dns.managedZones().list(projectId).execute().getManagedZones().stream()
+          .findAny()
+          .ifPresent(zone -> logger.atInfo().log("Found one zone [%s].", zone.getName()));
+      logger.atInfo().log("Finished accessing dns.");
+    } catch (IOException ioe) {
+      logger.atSevere().withCause(ioe).log("Failed to access dns.");
+    }
+    try {
+      CloudTasksClient client = component.cloudtasksClient();
+      com.google.cloud.tasks.v2.Queue queue =
+          client.getQueue(
+              String.format(
+                  "projects/%s/locations/%s/queues/async-actions",
+                  projectId, component.locationId()));
+      logger.atInfo().log("Got async queue state [%s]", queue.getState().name());
+      logger.atInfo().log("Finished accessing cloudtasks.");
+    } catch (RuntimeException e) {
+      logger.atSevere().withCause(e).log("Failed to access cloudtasks.");
+    }
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        ConfigModule.class,
+        CredentialModule.class,
+        CannedScriptsModule.class,
+        UtilsModule.class
+      })
+  interface CannedScriptsComponent {
+    Bigquery bigQuery();
+
+    CloudTasksClient cloudtasksClient();
+
+    Dataflow dataflow();
+
+    Dns dns();
+
+    Storage gcs();
+
+    @Config("projectId")
+    String projectId();
+
+    @Config("locationId")
+    String locationId();
+  }
+
+  @Module
+  static class CannedScriptsModule {
+    @Provides
+    static Bigquery provideBigquery(
+        @ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
+        @Config("projectId") String projectId) {
+      return new Bigquery.Builder(
+              credentialsBundle.getHttpTransport(),
+              credentialsBundle.getJsonFactory(),
+              credentialsBundle.getHttpRequestInitializer())
+          .setApplicationName(projectId)
+          .build();
+    }
+
+    @Provides
+    static Dataflow provideDataflow(
+        @ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
+        @Config("projectId") String projectId) {
+      return new Dataflow.Builder(
+              credentialsBundle.getHttpTransport(),
+              credentialsBundle.getJsonFactory(),
+              credentialsBundle.getHttpRequestInitializer())
+          .setApplicationName(String.format("%s billing", projectId))
+          .build();
+    }
+
+    @Provides
+    static Storage provideGcs(
+        @ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle) {
+      return StorageOptions.newBuilder()
+          .setCredentials(credentialsBundle.getGoogleCredentials())
+          .build()
+          .getService();
+    }
+
+    @Provides
+    static Dns provideDns(
+        @ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
+        @Config("projectId") String projectId,
+        @Config("cloudDnsRootUrl") Optional<String> rootUrl,
+        @Config("cloudDnsServicePath") Optional<String> servicePath) {
+      Dns.Builder builder =
+          new Dns.Builder(
+                  credentialsBundle.getHttpTransport(),
+                  credentialsBundle.getJsonFactory(),
+                  credentialsBundle.getHttpRequestInitializer())
+              .setApplicationName(projectId);
+
+      rootUrl.ifPresent(builder::setRootUrl);
+      servicePath.ifPresent(builder::setServicePath);
+
+      return builder.build();
+    }
+
+    @Provides
+    public static CloudTasksClient provideCloudTasksClient(
+        @ApplicationDefaultCredential GoogleCredentialsBundle credentials) {
+      CloudTasksClient client;
+      try {
+        client =
+            CloudTasksClient.create(
+                CloudTasksSettings.newBuilder()
+                    .setCredentialsProvider(
+                        FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
+                    .build());
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+      return client;
+    }
+  }
+}

core/src/main/java/google/registry/batch/cannedscript/GroupsApiChecker.java

@@ -1,122 +0,0 @@
-// Copyright 2022 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.batch.cannedscript;
-
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.util.RegistrarUtils.normalizeRegistrarId;
-
-import com.google.api.services.admin.directory.Directory;
-import com.google.api.services.groupssettings.Groupssettings;
-import com.google.common.base.Supplier;
-import com.google.common.base.Suppliers;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Streams;
-import com.google.common.flogger.FluentLogger;
-import dagger.Component;
-import dagger.Module;
-import dagger.Provides;
-import google.registry.config.CredentialModule;
-import google.registry.config.CredentialModule.AdcDelegatedCredential;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.config.RegistryConfig.ConfigModule;
-import google.registry.groups.DirectoryGroupsConnection;
-import google.registry.model.registrar.Registrar;
-import google.registry.model.registrar.RegistrarPoc;
-import google.registry.util.GoogleCredentialsBundle;
-import google.registry.util.UtilsModule;
-import java.util.List;
-import java.util.Set;
-import javax.inject.Singleton;
-
-/**
- * Verifies that the credential with the {@link AdcDelegatedCredential} annotation can be used to
- * access the Google Workspace Groups API.
- */
-// TODO(b/234424397): remove class after credential changes are rolled out.
-public class GroupsApiChecker {
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
-  private static final Supplier<GroupsConnectionComponent> COMPONENT_SUPPLIER =
-      Suppliers.memoize(DaggerGroupsApiChecker_GroupsConnectionComponent::create);
-
-  public static void runGroupsApiChecks() {
-    GroupsConnectionComponent component = COMPONENT_SUPPLIER.get();
-    DirectoryGroupsConnection groupsConnection = component.groupsConnection();
-
-    List<Registrar> registrars =
-        Streams.stream(Registrar.loadAllCached())
-            .filter(registrar -> registrar.isLive() && registrar.getType() == Registrar.Type.REAL)
-            .collect(toImmutableList());
-    for (Registrar registrar : registrars) {
-      for (final RegistrarPoc.Type type : RegistrarPoc.Type.values()) {
-        String groupKey =
-            String.format(
-                "%s-%s-contacts@%s",
-                normalizeRegistrarId(registrar.getRegistrarId()),
-                type.getDisplayName(),
-                component.gSuiteDomainName());
-        try {
-          Set<String> currentMembers = groupsConnection.getMembersOfGroup(groupKey);
-          logger.atInfo().log("Found %s members for %s.", currentMembers.size(), groupKey);
-        } catch (Exception e) {
-          Throwables.throwIfUnchecked(e);
-          throw new RuntimeException(e);
-        }
-      }
-    }
-  }
-
-  @Singleton
-  @Component(
-      modules = {
-        ConfigModule.class,
-        CredentialModule.class,
-        GroupsApiModule.class,
-        UtilsModule.class
-      })
-  interface GroupsConnectionComponent {
-    DirectoryGroupsConnection groupsConnection();
-
-    @Config("gSuiteDomainName")
-    String gSuiteDomainName();
-  }
-
-  @Module
-  static class GroupsApiModule {
-    @Provides
-    static Directory provideDirectory(
-        @AdcDelegatedCredential GoogleCredentialsBundle credentialsBundle,
-        @Config("projectId") String projectId) {
-      return new Directory.Builder(
-              credentialsBundle.getHttpTransport(),
-              credentialsBundle.getJsonFactory(),
-              credentialsBundle.getHttpRequestInitializer())
-          .setApplicationName(projectId)
-          .build();
-    }
-
-    @Provides
-    static Groupssettings provideGroupsSettings(
-        @AdcDelegatedCredential GoogleCredentialsBundle credentialsBundle,
-        @Config("projectId") String projectId) {
-      return new Groupssettings.Builder(
-              credentialsBundle.getHttpTransport(),
-              credentialsBundle.getJsonFactory(),
-              credentialsBundle.getHttpRequestInitializer())
-          .setApplicationName(projectId)
-          .build();
-    }
-  }
-}

core/src/main/java/google/registry/beam/BeamUtils.java

@@ -18,9 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
-import com.google.common.io.Resources;
 import google.registry.util.Clock;
-import google.registry.util.ResourceUtils;
 import java.util.regex.Pattern;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
@@ -57,14 +55,6 @@ public class BeamUtils {
     }
   }
 
-  /**
-   * Returns the {@link String} contents for a file in the {@code sql/} directory relative to a
-   * class.
-   */
-  public static String getQueryFromFile(Class<?> clazz, String filename) {
-    return ResourceUtils.readResourceUtf8(Resources.getResource(clazz, "sql/" + filename));
-  }
-
   /** Creates a beam job name and validates that it conforms to the requirements. */
   public static String createJobName(String prefix, Clock clock) {
     // Flex template job name must be unique and consists of only characters [-a-z0-9], starting

core/src/main/java/google/registry/beam/billing/BillingEvent.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.beam.invoicing;
+package google.registry.beam.billing;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.base.Joiner;
@@ -64,7 +64,7 @@ public abstract class BillingEvent implements Serializable {
           "amount",
           "flags");
 
-  /** Returns the unique ID for the {@code OneTime} associated with this event. */
+  /** Returns the unique ID for the {@code BillingEvent} associated with this event. */
   abstract long id();
 
   /** Returns the UTC DateTime this event becomes billable. */

core/src/main/java/google/registry/beam/billing/ExpandBillingRecurrencesPipeline.java

@@ -0,0 +1,508 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.billing;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Sets.difference;
+import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
+import static google.registry.model.domain.Period.Unit.YEARS;
+import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.CollectionUtils.union;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static google.registry.util.DateTimeUtils.earliestOf;
+import static google.registry.util.DateTimeUtils.latestOf;
+import static org.apache.beam.sdk.values.TypeDescriptors.voids;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Range;
+import dagger.Component;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.flows.custom.CustomLogicFactoryModule;
+import google.registry.flows.custom.CustomLogicModule;
+import google.registry.flows.domain.DomainPricingLogic;
+import google.registry.flows.domain.DomainPricingLogic.AllocationTokenInvalidForPremiumNameException;
+import google.registry.model.ImmutableObject;
+import google.registry.model.billing.BillingBase.Flag;
+import google.registry.model.billing.BillingCancellation;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingRecurrence;
+import google.registry.model.common.Cursor;
+import google.registry.model.domain.Domain;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.Period;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
+import google.registry.model.tld.Tld;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.util.Clock;
+import google.registry.util.SystemClock;
+import java.io.Serializable;
+import java.math.BigInteger;
+import java.util.Optional;
+import java.util.Set;
+import javax.inject.Singleton;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.GroupIntoBatches;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Wait;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.DateTime;
+
+/**
+ * Definition of a Dataflow Flex pipeline template, which expands {@link BillingRecurrence} to
+ * {@link BillingEvent} when an autorenew occurs within the given time frame.
+ *
+ * <p>This pipeline works in three stages:
+ *
+ * <ul>
+ *   <li>Gather the {@link BillingRecurrence}s that are in scope for expansion. The exact condition
+ *       of {@link BillingRecurrence}s to include can be found in {@link
+ *       #getRecurrencesInScope(Pipeline)}.
+ *   <li>Expand the {@link BillingRecurrence}s to {@link BillingEvent} (and corresponding {@link
+ *       DomainHistory}) that fall within the [{@link #startTime}, {@link #endTime}) window,
+ *       excluding those that are already present (to make this pipeline idempotent when running
+ *       with the same parameters multiple times, either in parallel or in sequence). The {@link
+ *       BillingRecurrence} is also updated with the information on when it was last expanded, so it
+ *       would not be in scope for expansion until at least a year later.
+ *   <li>If the cursor for billing events should be advanced, advance it to {@link #endTime} after
+ *       all of the expansions in the previous step is done, only when it is currently at {@link
+ *       #startTime}.
+ * </ul>
+ *
+ * <p>Note that the creation of new {@link BillingEvent} and {@link DomainHistory} is done
+ * speculatively as soon as its event time is in scope for expansion (i.e. within the window of
+ * operation). If a domain is subsequently cancelled during the autorenew grace period, a {@link
+ * BillingCancellation} would have been created to cancel the {@link BillingEvent} out. Similarly, a
+ * {@link DomainHistory} for the delete will be created which negates the effect of the
+ * speculatively created {@link DomainHistory}, specifically for the transaction records. Both the
+ * {@link BillingEvent} and {@link DomainHistory} will only be used (and cancelled out) when the
+ * billing time becomes effective, which is after the grace period, when the cancellations would
+ * have been written, if need be. This is no different from what we do with manual renewals or
+ * normal creates, where entities are always created for the action regardless of whether their
+ * effects will be negated later due to subsequent actions within respective grace periods.
+ *
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha \
+ * --pipeline=expandBilling}.
+ *
+ * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
+ *
+ * @see BillingCancellation#forGracePeriod
+ * @see google.registry.flows.domain.DomainFlowUtils#createCancelingRecords
+ * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
+ *     Flex Templates</a>
+ */
+public class ExpandBillingRecurrencesPipeline implements Serializable {
+
+  private static final long serialVersionUID = -5827984301386630194L;
+
+  private static final DomainPricingLogic domainPricingLogic;
+
+  private static final int batchSize;
+
+  static {
+    PipelineComponent pipelineComponent =
+        DaggerExpandBillingRecurrencesPipeline_PipelineComponent.create();
+    domainPricingLogic = pipelineComponent.domainPricingLogic();
+    batchSize = pipelineComponent.batchSize();
+  }
+
+  // Inclusive lower bound of the expansion window.
+  private final DateTime startTime;
+  // Exclusive lower bound of the expansion window.
+  private final DateTime endTime;
+  private final boolean isDryRun;
+  private final boolean advanceCursor;
+  private final Counter recurrencesInScopeCounter =
+      Metrics.counter("ExpandBilling", "Recurrences in scope for expansion");
+  // Note that this counter is only accurate when running in dry run mode. Because SQL persistence
+  // is a side effect and not idempotent, a transaction to save OneTimes could be successful but the
+  // transform that contains it could be still be retried, rolling back the counter increment. The
+  // same transform, when retried, would skip the already expanded OneTime, causing this counter to
+  // be lower than it should be when not in dry run mode.
+  // See: https://beam.apache.org/documentation/programming-guide/#user-code-idempotence
+  private final Counter oneTimesToExpandCounter =
+      Metrics.counter("ExpandBilling", "OneTimes that would be expanded");
+
+  ExpandBillingRecurrencesPipeline(ExpandBillingRecurrencesPipelineOptions options, Clock clock) {
+    startTime = DateTime.parse(options.getStartTime());
+    endTime = DateTime.parse(options.getEndTime());
+    checkArgument(
+        !endTime.isAfter(clock.nowUtc()),
+        String.format("End time %s must be at or before now.", endTime));
+    checkArgument(
+        startTime.isBefore(endTime),
+        String.format("[%s, %s) is not a valid window of operation.", startTime, endTime));
+    isDryRun = options.getIsDryRun();
+    advanceCursor = options.getAdvanceCursor();
+  }
+
+  private PipelineResult run(Pipeline pipeline) {
+    setupPipeline(pipeline);
+    return pipeline.run();
+  }
+
+  void setupPipeline(Pipeline pipeline) {
+    PCollection<KV<Integer, Long>> recurrenceIds = getRecurrencesInScope(pipeline);
+    PCollection<Void> expanded = expandRecurrences(recurrenceIds);
+    if (!isDryRun && advanceCursor) {
+      advanceCursor(expanded);
+    }
+  }
+
+  PCollection<KV<Integer, Long>> getRecurrencesInScope(Pipeline pipeline) {
+    return pipeline.apply(
+        "Read all Recurrences in scope",
+        // Use native query because JPQL does not support timestamp arithmetics.
+        RegistryJpaIO.read(
+                "SELECT billing_recurrence_id "
+                    + "FROM \"BillingRecurrence\" "
+                    // Recurrence should not close before the first event time.
+                    + "WHERE event_time < recurrence_end_time "
+                    // First event time should be before end time.
+                    + "AND event_Time < :endTime "
+                    // Recurrence should not close before start time.
+                    + "AND :startTime < recurrence_end_time "
+                    // Last expansion should happen at least one year before end time.
+                    + "AND recurrence_last_expansion < :oneYearAgo "
+                    // The recurrence should not close before next expansion time.
+                    + "AND recurrence_last_expansion + INTERVAL '1 YEAR' < recurrence_end_time",
+                ImmutableMap.of(
+                    "endTime",
+                    endTime,
+                    "startTime",
+                    startTime,
+                    "oneYearAgo",
+                    endTime.minusYears(1)),
+                true,
+                (BigInteger id) -> {
+                  recurrencesInScopeCounter.inc();
+                  // Note that because all elements are mapped to the same dummy key, the next
+                  // batching transform will effectively be serial. This however does not matter for
+                  // our use case because the elements were obtained from a SQL read query, which
+                  // are returned sequentially already. Therefore, having a sequential step to group
+                  // them does not reduce overall parallelism of the pipeline, and the batches can
+                  // then be distributed to all available workers for further processing, where the
+                  // main benefit of parallelism shows. In benchmarking, turning the distribution
+                  // of elements in this step resulted in marginal improvement in overall
+                  // performance at best without clear indication on why or to which degree. If the
+                  // runtime becomes a concern later on, we could consider fine-tuning the sharding
+                  // of output elements in this step.
+                  //
+                  // See: https://stackoverflow.com/a/44956702/791306
+                  return KV.of(0, id.longValue());
+                })
+            .withCoder(KvCoder.of(VarIntCoder.of(), VarLongCoder.of())));
+  }
+
+  private PCollection<Void> expandRecurrences(PCollection<KV<Integer, Long>> recurrenceIds) {
+    return recurrenceIds
+        .apply(
+            "Group into batches",
+            GroupIntoBatches.<Integer, Long>ofSize(batchSize).withShardedKey())
+        .apply(
+            "Expand and save Recurrences into OneTimes and corresponding DomainHistories",
+            MapElements.into(voids())
+                .via(
+                    element -> {
+                      Iterable<Long> ids = element.getValue();
+                      tm().transact(
+                              () -> {
+                                ImmutableSet.Builder<ImmutableObject> results =
+                                    new ImmutableSet.Builder<>();
+                                ids.forEach(id -> expandOneRecurrence(id, results));
+                                if (!isDryRun) {
+                                  tm().putAll(results.build());
+                                }
+                              });
+                      return null;
+                    }));
+  }
+
+  private void expandOneRecurrence(
+      Long recurrenceId, ImmutableSet.Builder<ImmutableObject> results) {
+    BillingRecurrence billingRecurrence =
+        tm().loadByKey(BillingRecurrence.createVKey(recurrenceId));
+
+    // Determine the complete set of EventTimes this recurrence event should expand to within
+    // [max(recurrenceLastExpansion + 1 yr, startTime), min(recurrenceEndTime, endTime)).
+    //
+    // This range should always be legal for recurrences that are returned from the query. However,
+    // it is possible that the recurrence has changed between when the read transformation occurred
+    // and now. This could be caused by some out-of-process mutations (such as a domain deletion
+    // closing out a previously open-ended recurrence), or more subtly, Beam could execute the same
+    // work multiple times due to transient communication issues between workers and the scheduler.
+    // Such opportunistic retries are OK for pure functional transformations, but can cause
+    // unexpected behavior when side effects are executed more than once. For example, the
+    // recurrence_last_expansion field could be updated by a worker after a success expansion, which
+    // failed to report the status to the scheduler in time, which in turn scheduled another worker
+    // to work on the same batch. The second worker would see a new recurrence_last_expansion that
+    // causes the range to be illegal.
+    //
+    // The best way to handle any unexpected behavior is to simply drop the recurrence from
+    // expansion, if its new state still calls for an expansion, it would be picked up the next time
+    // the pipeline runs.
+    ImmutableSet<DateTime> eventTimes;
+    try {
+      eventTimes =
+          ImmutableSet.copyOf(
+              billingRecurrence
+                  .getRecurrenceTimeOfYear()
+                  .getInstancesInRange(
+                      Range.closedOpen(
+                          latestOf(
+                              billingRecurrence.getRecurrenceLastExpansion().plusYears(1),
+                              startTime),
+                          earliestOf(billingRecurrence.getRecurrenceEndTime(), endTime))));
+    } catch (IllegalArgumentException e) {
+      return;
+    }
+    Domain domain = tm().loadByKey(Domain.createVKey(billingRecurrence.getDomainRepoId()));
+    Tld tld = Tld.get(domain.getTld());
+
+    // Find the times for which the OneTime billing event are already created, making this expansion
+    // idempotent. There is no need to match to the domain repo ID as the cancellation matching
+    // billing event itself can only be for a single domain.
+    ImmutableSet<DateTime> existingEventTimes =
+        ImmutableSet.copyOf(
+            tm().query(
+                    "SELECT eventTime FROM BillingEvent WHERE cancellationMatchingBillingEvent ="
+                        + " :key",
+                    DateTime.class)
+                .setParameter("key", billingRecurrence.createVKey())
+                .getResultList());
+
+    Set<DateTime> eventTimesToExpand = difference(eventTimes, existingEventTimes);
+
+    if (eventTimesToExpand.isEmpty()) {
+      return;
+    }
+
+    DateTime recurrenceLastExpansionTime = billingRecurrence.getRecurrenceLastExpansion();
+
+    // Create new OneTime and DomainHistory for EventTimes that needs to be expanded.
+    for (DateTime eventTime : eventTimesToExpand) {
+      recurrenceLastExpansionTime = latestOf(recurrenceLastExpansionTime, eventTime);
+      oneTimesToExpandCounter.inc();
+      DateTime billingTime = eventTime.plus(tld.getAutoRenewGracePeriodLength());
+      // Note that the DomainHistory is created as of transaction time, as opposed to event time.
+      // This might be counterintuitive because other DomainHistories are created at the time
+      // mutation events occur, such as in DomainDeleteFlow or DomainRenewFlow. Therefore, it is
+      // possible to have a DomainHistory for a delete during the autorenew grace period with a
+      // modification time before that of the DomainHistory for the autorenew itself. This is not
+      // ideal, but necessary because we save the **current** state of the domain (as of transaction
+      // time) to the DomainHistory , instead of the state of the domain as of event time (which
+      // would required loading the domain from DomainHistory at event time).
+      //
+      // Even though doing the loading is seemly possible, it generally is a bad idea to create
+      // DomainHistories retroactively and in all instances that we create a HistoryEntry we always
+      // set the modification time to the transaction time. It would also violate the invariance
+      // that a DomainHistory with a higher revision ID (which is always allocated with monotonic
+      // increase) always has a later modification time.
+      //
+      // Lastly because the domain entity itself did not change as part of the expansion, we should
+      // not project it to transaction time before saving it in the history, which would require us
+      // to save the projected domain as well. Any changes to the domain itself are handled when
+      // the domain is actually used or explicitly projected and saved. The DomainHistory created
+      // here does not actually affect anything materially (e.g. RDE). We can understand it in such
+      // a way that this history represents not when the domain is autorenewed (at event time), but
+      // when its autorenew billing event is created (at transaction time).
+      DomainHistory historyEntry =
+          new DomainHistory.Builder()
+              .setBySuperuser(false)
+              .setRegistrarId(billingRecurrence.getRegistrarId())
+              .setModificationTime(tm().getTransactionTime())
+              .setDomain(domain)
+              .setPeriod(Period.create(1, YEARS))
+              .setReason("Domain autorenewal by ExpandRecurringBillingEventsPipeline")
+              .setRequestedByRegistrar(false)
+              .setType(DOMAIN_AUTORENEW)
+              .setDomainTransactionRecords(
+                  // Don't write a domain transaction record if the domain is deleted before billing
+                  // time (i.e. within the autorenew grace period). We cannot rely on a negating
+                  // DomainHistory created by DomainDeleteFlow because it only cancels transaction
+                  // records already present. In this case the domain was deleted before this
+                  // pipeline runs to expand the OneTime (which should be rare because this pipeline
+                  // should run every day), and no negating transaction records would have been
+                  // created when the deletion occurred. Again, there is no need to project the
+                  // domain, because if it were deleted before this transaction, its updated delete
+                  // time would have already been loaded here.
+                  //
+                  // We don't compare recurrence end time with billing time because the recurrence
+                  // could be caused for other reasons during the grace period, like a manual
+                  // renewal, in which case we still want to write the transaction record. Also,
+                  // the expansion happens when event time is in scope, which means the billing time
+                  // is still 45 days in the future, and the recurrence could have been closed
+                  // between now and then.
+                  //
+                  // A side effect of this logic is that if a transfer occurs within the ARGP, it
+                  // would have recorded both a TRANSFER_SUCCESSFUL and a NET_RENEWS_1_YEAR, even
+                  // though the transfer would have subsumed the autorenew. There is no perfect
+                  // solution for this because even if we expand the recurrence when the billing
+                  // event is in scope (as was the case in the old action), we still cannot use
+                  // recurrence end time < billing time as an indicator for if a transfer had
+                  // occurred during ARGP (see last paragraph, renewals during ARGP also close the
+                  // recurrence),therefore we still cannot always be correct when constructing the
+                  // transaction records that way (either we miss transfers, or we miss renewals
+                  // during ARGP).
+                  //
+                  // See: DomainFlowUtils#createCancellingRecords
+                  domain.getDeletionTime().isBefore(billingTime)
+                      ? ImmutableSet.of()
+                      : ImmutableSet.of(
+                          DomainTransactionRecord.create(
+                              tld.getTldStr(),
+                              // We report this when the autorenew grace period ends.
+                              billingTime,
+                              TransactionReportField.netRenewsFieldFromYears(1),
+                              1)))
+              .build();
+      results.add(historyEntry);
+
+      // It is OK to always create a OneTime, even though the domain might be deleted or transferred
+      // later during autorenew grace period, as a cancellation will always be written out in those
+      // instances.
+      BillingEvent billingEvent = null;
+      try {
+        billingEvent =
+            new BillingEvent.Builder()
+                .setBillingTime(billingTime)
+                .setRegistrarId(billingRecurrence.getRegistrarId())
+                // Determine the cost for a one-year renewal.
+                .setCost(
+                    domainPricingLogic
+                        .getRenewPrice(
+                            tld,
+                            billingRecurrence.getTargetId(),
+                            eventTime,
+                            1,
+                            billingRecurrence,
+                            Optional.empty())
+                        .getRenewCost())
+                .setEventTime(eventTime)
+                .setFlags(union(billingRecurrence.getFlags(), Flag.SYNTHETIC))
+                .setDomainHistory(historyEntry)
+                .setPeriodYears(1)
+                .setReason(billingRecurrence.getReason())
+                .setSyntheticCreationTime(endTime)
+                .setCancellationMatchingBillingEvent(billingRecurrence)
+                .setTargetId(billingRecurrence.getTargetId())
+                .build();
+      } catch (AllocationTokenInvalidForPremiumNameException e) {
+        // This should not be reached since we are not using an allocation token
+        return;
+      }
+      results.add(billingEvent);
+    }
+    results.add(
+        billingRecurrence
+            .asBuilder()
+            .setRecurrenceLastExpansion(recurrenceLastExpansionTime)
+            .build());
+  }
+
+  private PDone advanceCursor(PCollection<Void> persisted) {
+    return PDone.in(
+        persisted
+            .getPipeline()
+            .apply("Create one dummy element", Create.of((Void) null))
+            .apply("Wait for all saves to finish", Wait.on(persisted))
+            // Because only one dummy element is created in the start PCollection, this
+            // transform is guaranteed to only process one element and therefore only run once.
+            // Because the previous step waits for all emissions of voids from the expansion step to
+            // finish, this transform is guaranteed to run only after all expansions are done and
+            // persisted.
+            .apply(
+                "Advance cursor",
+                ParDo.of(
+                    new DoFn<Void, Void>() {
+                      @ProcessElement
+                      public void processElement() {
+                        tm().transact(
+                                () -> {
+                                  DateTime currentCursorTime =
+                                      tm().loadByKeyIfPresent(
+                                              Cursor.createGlobalVKey(RECURRING_BILLING))
+                                          .orElse(
+                                              Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
+                                          .getCursorTime();
+                                  if (!currentCursorTime.equals(startTime)) {
+                                    throw new IllegalStateException(
+                                        String.format(
+                                            "Current cursor position %s does not match start time"
+                                                + " %s.",
+                                            currentCursorTime, startTime));
+                                  }
+                                  tm().put(Cursor.createGlobal(RECURRING_BILLING, endTime));
+                                });
+                      }
+                    }))
+            .getPipeline());
+  }
+
+  public static void main(String[] args) {
+    PipelineOptionsFactory.register(ExpandBillingRecurrencesPipelineOptions.class);
+    ExpandBillingRecurrencesPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args)
+            .withValidation()
+            .as(ExpandBillingRecurrencesPipelineOptions.class);
+    // Hardcode the transaction level to be at serializable we do not want concurrent runs of the
+    // pipeline for the same window to create duplicate OneTimes. This ensures that the set of
+    // existing OneTimes do not change by the time new OneTimes are inserted within a transaction.
+    //
+    // Per PostgreSQL, serializable isolation level does not introduce any blocking beyond that
+    // present in  repeatable read other than some overhead related to monitoring possible
+    // serializable anomalies. Therefore, in most cases, since each worker of the same job works on
+    // a different set of recurrences, it is not possible for their execution order to affect
+    // serialization outcome, and the performance penalty should be minimum when using serializable
+    // compared to using repeatable read.
+    //
+    // We should pay some attention to the runtime of the job and logs when we run this job daily on
+    // production to check the actual performance impact for using this isolation level (i.e. check
+    // the frequency of occurrence of retried transactions due to serialization errors) to assess
+    // the actual parallelism of the job.
+    //
+    // See: https://www.postgresql.org/docs/current/transaction-iso.html
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE);
+    Pipeline pipeline = Pipeline.create(options);
+    new ExpandBillingRecurrencesPipeline(options, new SystemClock()).run(pipeline);
+  }
+
+  @Singleton
+  @Component(
+      modules = {CustomLogicModule.class, CustomLogicFactoryModule.class, ConfigModule.class})
+  interface PipelineComponent {
+
+    DomainPricingLogic domainPricingLogic();
+
+    @Config("jdbcBatchSize")
+    int batchSize();
+  }
+}

core/src/main/java/google/registry/beam/billing/ExpandBillingRecurrencesPipelineOptions.java

@@ -0,0 +1,49 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.billing;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+
+public interface ExpandBillingRecurrencesPipelineOptions extends RegistryPipelineOptions {
+  @Description(
+      "The inclusive lower bound of on the range of event times that will be expanded, in ISO 8601"
+          + " format")
+  String getStartTime();
+
+  void setStartTime(String startTime);
+
+  @Description(
+      "The exclusive upper bound of on the range of event times that will be expanded, in ISO 8601"
+          + " format")
+  String getEndTime();
+
+  void setEndTime(String endTime);
+
+  @Description("If true, the expanded billing events and history entries will not be saved.")
+  @Default.Boolean(false)
+  boolean getIsDryRun();
+
+  void setIsDryRun(boolean isDryRun);
+
+  @Description(
+      "If true, set the RECURRING_BILLING global cursor to endTime after saving all expanded"
+          + " billing events and history entries.")
+  @Default.Boolean(true)
+  boolean getAdvanceCursor();
+
+  void setAdvanceCursor(boolean advanceCursor);
+}

core/src/main/java/google/registry/beam/billing/InvoicingPipeline.java

@@ -12,23 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.beam.invoicing;
+package google.registry.beam.billing;
 
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.beam.BeamUtils.getQueryFromFile;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
 
 import com.google.common.flogger.FluentLogger;
+import google.registry.beam.billing.BillingEvent.InvoiceGroupingKey;
+import google.registry.beam.billing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
 import google.registry.beam.common.RegistryJpaIO;
 import google.registry.beam.common.RegistryJpaIO.Read;
-import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey;
-import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
-import google.registry.model.billing.BillingEvent.Flag;
-import google.registry.model.billing.BillingEvent.OneTime;
+import google.registry.model.billing.BillingBase.Flag;
+import google.registry.model.billing.BillingEvent;
 import google.registry.model.registrar.Registrar;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.reporting.billing.BillingModule;
 import google.registry.util.DomainNameUtils;
+import google.registry.util.ResourceUtils;
 import google.registry.util.SqlTemplate;
 import java.io.Serializable;
 import java.time.YearMonth;
@@ -86,29 +86,30 @@ public class InvoicingPipeline implements Serializable {
 
   void setupPipeline(Pipeline pipeline) {
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
-    PCollection<BillingEvent> billingEvents = readFromCloudSql(options, pipeline);
+    PCollection<google.registry.beam.billing.BillingEvent> billingEvents =
+        readFromCloudSql(options, pipeline);
     saveInvoiceCsv(billingEvents, options);
     saveDetailedCsv(billingEvents, options);
   }
 
-  static PCollection<BillingEvent> readFromCloudSql(
+  static PCollection<google.registry.beam.billing.BillingEvent> readFromCloudSql(
       InvoicingPipelineOptions options, Pipeline pipeline) {
-    Read<Object[], BillingEvent> read =
-        RegistryJpaIO.<Object[], BillingEvent>read(
+    Read<Object[], google.registry.beam.billing.BillingEvent> read =
+        RegistryJpaIO.<Object[], google.registry.beam.billing.BillingEvent>read(
                 makeCloudSqlQuery(options.getYearMonth()), false, row -> parseRow(row).orElse(null))
-            .withCoder(SerializableCoder.of(BillingEvent.class));
+            .withCoder(SerializableCoder.of(google.registry.beam.billing.BillingEvent.class));
 
-    PCollection<BillingEvent> billingEventsWithNulls =
+    PCollection<google.registry.beam.billing.BillingEvent> billingEventsWithNulls =
         pipeline.apply("Read BillingEvents from Cloud SQL", read);
 
     // Remove null billing events
     return billingEventsWithNulls.apply(Filter.by(Objects::nonNull));
   }
 
-  private static Optional<BillingEvent> parseRow(Object[] row) {
-    OneTime oneTime = (OneTime) row[0];
+  private static Optional<google.registry.beam.billing.BillingEvent> parseRow(Object[] row) {
+    BillingEvent billingEvent = (BillingEvent) row[0];
     Registrar registrar = (Registrar) row[1];
-    CurrencyUnit currency = oneTime.getCost().getCurrencyUnit();
+    CurrencyUnit currency = billingEvent.getCost().getCurrencyUnit();
     if (!registrar.getBillingAccountMap().containsKey(currency)) {
       logger.atSevere().log(
           "Registrar %s does not have a product account key for the currency unit: %s",
@@ -117,37 +118,40 @@ public class InvoicingPipeline implements Serializable {
     }
 
     return Optional.of(
-        BillingEvent.create(
-            oneTime.getId(),
-            oneTime.getBillingTime(),
-            oneTime.getEventTime(),
+        google.registry.beam.billing.BillingEvent.create(
+            billingEvent.getId(),
+            billingEvent.getBillingTime(),
+            billingEvent.getEventTime(),
             registrar.getRegistrarId(),
             registrar.getBillingAccountMap().get(currency),
             registrar.getPoNumber().orElse(""),
-            DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),
-            oneTime.getReason().toString(),
-            oneTime.getTargetId(),
-            oneTime.getDomainRepoId(),
-            Optional.ofNullable(oneTime.getPeriodYears()).orElse(0),
-            oneTime.getCost().getCurrencyUnit().toString(),
-            oneTime.getCost().getAmount().doubleValue(),
+            DomainNameUtils.getTldFromDomainName(billingEvent.getTargetId()),
+            billingEvent.getReason().toString(),
+            billingEvent.getTargetId(),
+            billingEvent.getDomainRepoId(),
+            Optional.ofNullable(billingEvent.getPeriodYears()).orElse(0),
+            billingEvent.getCost().getCurrencyUnit().toString(),
+            billingEvent.getCost().getAmount().doubleValue(),
             String.join(
-                " ", oneTime.getFlags().stream().map(Flag::toString).collect(toImmutableSet()))));
+                " ",
+                billingEvent.getFlags().stream().map(Flag::toString).collect(toImmutableSet()))));
   }
 
   /** Transform that converts a {@code BillingEvent} into an invoice CSV row. */
   private static class GenerateInvoiceRows
-      extends PTransform<PCollection<BillingEvent>, PCollection<String>> {
+      extends PTransform<
+          PCollection<google.registry.beam.billing.BillingEvent>, PCollection<String>> {
 
     private static final long serialVersionUID = -8090619008258393728L;
 
     @Override
-    public PCollection<String> expand(PCollection<BillingEvent> input) {
+    public PCollection<String> expand(
+        PCollection<google.registry.beam.billing.BillingEvent> input) {
       return input
           .apply(
               "Map to invoicing key",
               MapElements.into(TypeDescriptor.of(InvoiceGroupingKey.class))
-                  .via(BillingEvent::getInvoiceGroupingKey))
+                  .via(google.registry.beam.billing.BillingEvent::getInvoiceGroupingKey))
           .apply(
               "Filter out free events", Filter.by((InvoiceGroupingKey key) -> key.unitPrice() != 0))
           .setCoder(new InvoiceGroupingKeyCoder())
@@ -161,7 +165,8 @@ public class InvoicingPipeline implements Serializable {
 
   /** Saves the billing events to a single overall invoice CSV file. */
   static void saveInvoiceCsv(
-      PCollection<BillingEvent> billingEvents, InvoicingPipelineOptions options) {
+      PCollection<google.registry.beam.billing.BillingEvent> billingEvents,
+      InvoicingPipelineOptions options) {
     billingEvents
         .apply("Generate overall invoice rows", new GenerateInvoiceRows())
         .apply(
@@ -182,16 +187,17 @@ public class InvoicingPipeline implements Serializable {
 
   /** Saves the billing events to detailed report CSV files keyed by registrar-tld pairs. */
   static void saveDetailedCsv(
-      PCollection<BillingEvent> billingEvents, InvoicingPipelineOptions options) {
+      PCollection<google.registry.beam.billing.BillingEvent> billingEvents,
+      InvoicingPipelineOptions options) {
     String yearMonth = options.getYearMonth();
     billingEvents.apply(
         "Write detailed report for each registrar-tld pair",
-        FileIO.<String, BillingEvent>writeDynamic()
+        FileIO.<String, google.registry.beam.billing.BillingEvent>writeDynamic()
             .to(
                 String.format(
                     "%s/%s/%s",
                     options.getBillingBucketUrl(), BillingModule.INVOICES_DIRECTORY, yearMonth))
-            .by(BillingEvent::getDetailedReportGroupingKey)
+            .by(google.registry.beam.billing.BillingEvent::getDetailedReportGroupingKey)
             .withNumShards(1)
             .withDestinationCoder(StringUtf8Coder.of())
             .withNaming(
@@ -200,8 +206,8 @@ public class InvoicingPipeline implements Serializable {
                         String.format(
                             "%s_%s_%s.csv", BillingModule.DETAIL_REPORT_PREFIX, yearMonth, key))
             .via(
-                Contextful.fn(BillingEvent::toCsv),
-                TextIO.sink().withHeader(BillingEvent.getHeader())));
+                Contextful.fn(google.registry.beam.billing.BillingEvent::toCsv),
+                TextIO.sink().withHeader(google.registry.beam.billing.BillingEvent.getHeader())));
   }
 
   /** Create the Cloud SQL query for a given yearMonth at runtime. */
@@ -209,7 +215,8 @@ public class InvoicingPipeline implements Serializable {
     YearMonth endMonth = YearMonth.parse(yearMonth).plusMonths(1);
     String queryWithComments =
         SqlTemplate.create(
-                getQueryFromFile(InvoicingPipeline.class, "cloud_sql_billing_events.sql"))
+                ResourceUtils.readResourceUtf8(
+                    InvoicingPipeline.class, "sql/cloud_sql_billing_events.sql"))
             .put("FIRST_TIMESTAMP_OF_MONTH", yearMonth + "-01")
             .put(
                 "LAST_TIMESTAMP_OF_MONTH",

core/src/main/java/google/registry/beam/billing/InvoicingPipelineOptions.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.beam.invoicing;
+package google.registry.beam.billing;
 
 import google.registry.beam.common.RegistryPipelineOptions;
 import org.apache.beam.sdk.options.Description;

core/src/main/java/google/registry/beam/common/DatabaseSnapshot.java

@@ -15,7 +15,7 @@
 package google.registry.beam.common;
 
 import static com.google.common.base.Preconditions.checkState;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.flogger.FluentLogger;
 import java.util.List;
@@ -52,7 +52,7 @@ public class DatabaseSnapshot implements AutoCloseable {
   }
 
   private DatabaseSnapshot open() {
-    entityManager = jpaTm().getStandaloneEntityManager();
+    entityManager = tm().getStandaloneEntityManager();
     transaction = entityManager.getTransaction();
     transaction.setRollbackOnly();
     transaction.begin();

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -14,8 +14,7 @@
 
 package google.registry.beam.common;
 
-import static com.google.common.base.Preconditions.checkArgument;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.apache.beam.sdk.values.TypeDescriptors.integers;
 
 import com.google.auto.value.AutoValue;
@@ -26,7 +25,6 @@ import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.ThreadLocalRandom;
 import javax.annotation.Nullable;
 import javax.persistence.criteria.CriteriaQuery;
 import org.apache.beam.sdk.coders.Coder;
@@ -51,8 +49,8 @@ import org.apache.beam.sdk.values.PCollection;
  *
  * <p>The {@code JpaTransactionManager} is instantiated once on each pipeline worker VM (through
  * {@link RegistryPipelineWorkerInitializer}), made available through the static method {@link
- * TransactionManagerFactory#jpaTm()}, and is shared by all threads on the VM. Configuration is
- * through {@link RegistryPipelineOptions}.
+ * TransactionManagerFactory#tm()}, and is shared by all threads on the VM. Configuration is through
+ * {@link RegistryPipelineOptions}.
  */
 public final class RegistryJpaIO {
 
@@ -73,7 +71,7 @@ public final class RegistryJpaIO {
   }
 
   /**
-   * Returns a {@link Read} connector based on the given {@code jpql} query string.
+   * Returns a {@link Read} connector based on the given native or {@code jpql} query string.
    *
    * <p>User should take care to prevent sql-injection attacks.
    */
@@ -232,11 +230,10 @@ public final class RegistryJpaIO {
 
       @ProcessElement
       public void processElement(OutputReceiver<T> outputReceiver) {
-        jpaTm()
-            .transactNoRetry(
+        tm().transactNoRetry(
                 () -> {
                   if (snapshotId != null) {
-                    jpaTm().setDatabaseSnapshot(snapshotId);
+                    tm().setDatabaseSnapshot(snapshotId);
                   }
                   query.stream().map(resultMapper::apply).forEach(outputReceiver::output);
                 });
@@ -263,38 +260,11 @@ public final class RegistryJpaIO {
 
     public static final int DEFAULT_BATCH_SIZE = 1;
 
-    /** The default number of write shard. Please refer to {@link #shards} for more information. */
-    public static final int DEFAULT_SHARDS = 1;
-
     public abstract String name();
 
     /** Number of elements to be written in one call. */
     public abstract int batchSize();
 
-    /**
-     * The number of shards the output should be split into.
-     *
-     * <p>This value is a hint to the pipeline runner on the level of parallelism, and should be
-     * significantly greater than the number of threads working on this transformation (see next
-     * paragraph for more information). On the other hand, it should not be too large to the point
-     * that the number of elements per shard is lower than {@link #batchSize()}. As a rule of thumb,
-     * the following constraint should hold: {@code shards * batchSize * nThreads <=
-     * inputElementCount}. Although it is not always possible to determine the number of threads
-     * working on this transform, when the pipeline run is IO-bound, it most likely is close to the
-     * total number of threads in the pipeline, which is explained below.
-     *
-     * <p>With Cloud Dataflow runner, the total number of worker threads in a batch pipeline (which
-     * includes all existing Registry pipelines) is the number of vCPUs used by the pipeline, and
-     * can be set by the {@code --maxNumWorkers} and {@code --workerMachineType} parameters. The
-     * number of worker threads in a streaming pipeline can be set by the {@code --maxNumWorkers}
-     * and {@code --numberOfWorkerHarnessThreads} parameters.
-     *
-     * <p>Note that connections on the database server are a limited resource, therefore the number
-     * of threads that interact with the database should be set to an appropriate limit. Again, we
-     * cannot control this number, but can influence it by controlling the total number of threads.
-     */
-    public abstract int shards();
-
     public abstract SerializableFunction<T, Object> jpaConverter();
 
     public Write<T> withName(String name) {
@@ -305,10 +275,6 @@ public final class RegistryJpaIO {
       return toBuilder().batchSize(batchSize).build();
     }
 
-    public Write<T> withShards(int shards) {
-      return toBuilder().shards(shards).build();
-    }
-
     /**
      * An optional function that converts the input entities to a form that can be written into the
      * database.
@@ -322,10 +288,7 @@ public final class RegistryJpaIO {
     @Override
     public PCollection<Void> expand(PCollection<T> input) {
       return input
-          .apply(
-              "Shard data " + name(),
-              WithKeys.<Integer, T>of(e -> ThreadLocalRandom.current().nextInt(shards()))
-                  .withKeyType(integers()))
+          .apply("Add key to data " + name(), WithKeys.<Integer, T>of(0).withKeyType(integers()))
           // The call to withShardedKey() is performance critical. The resulting transform ensures
           // that data is spread evenly across all worker threads.
           .apply(
@@ -340,7 +303,6 @@ public final class RegistryJpaIO {
       return new AutoValue_RegistryJpaIO_Write.Builder<T>()
           .name(DEFAULT_NAME)
           .batchSize(DEFAULT_BATCH_SIZE)
-          .shards(DEFAULT_SHARDS)
           .jpaConverter(x -> x);
     }
 
@@ -351,8 +313,6 @@ public final class RegistryJpaIO {
 
       abstract Builder<T> batchSize(int batchSize);
 
-      abstract Builder<T> shards(int jdbcNumConnsHint);
-
       abstract Builder<T> jpaConverter(SerializableFunction<T, Object> jpaConverter);
 
       abstract Write<T> build();
@@ -382,12 +342,10 @@ public final class RegistryJpaIO {
               .filter(Objects::nonNull)
               .collect(ImmutableList.toImmutableList());
       try {
-        jpaTm()
-            .transact(
+        tm().transact(
                 () -> {
-                  // Don't modify existing objects as it could lead to race conditions
-                  entities.forEach(this::verifyObjectNonexistence);
-                  jpaTm().putAll(entities);
+                  // TODO(b/263502442): properly handle creations and blind-writes.
+                  tm().putAll(entities);
                 });
         counter.inc(entities.size());
       } catch (RuntimeException e) {
@@ -402,12 +360,10 @@ public final class RegistryJpaIO {
     private void processSingly(ImmutableList<Object> entities) {
       for (Object entity : entities) {
         try {
-          jpaTm()
-              .transact(
+          tm().transact(
                   () -> {
-                    // Don't modify existing objects as it could lead to race conditions
-                    verifyObjectNonexistence(entity);
-                    jpaTm().put(entity);
+                    // TODO(b/263502442): properly handle creations and blind-writes.
+                    tm().put(entity);
                   });
           counter.inc();
         } catch (RuntimeException e) {
@@ -419,14 +375,12 @@ public final class RegistryJpaIO {
     /** Returns this entity's primary key field(s) in a string. */
     private String toEntityKeyString(Object entity) {
       try {
-        return jpaTm()
-            .transact(
+        return tm().transact(
                 () ->
                     String.format(
                         "%s_%s",
                         entity.getClass().getSimpleName(),
-                        jpaTm()
-                            .getEntityManager()
+                        tm().getEntityManager()
                             .getEntityManagerFactory()
                             .getPersistenceUnitUtil()
                             .getIdentifier(entity)));
@@ -434,16 +388,5 @@ public final class RegistryJpaIO {
         return "Non-SqlEntity: " + entity;
       }
     }
-
-    /** SqlBatchWriter should not re-write existing entities due to potential race conditions. */
-    private void verifyObjectNonexistence(Object obj) {
-      // We cannot rely on calling "insert" on the objects because the underlying JPA persist call
-      // adds the input object to the persistence context, meaning that any modifications (e.g.
-      // updateTimestamp) are reflected in the input object. Beam doesn't allow modification of
-      // input objects, so this throws an exception.
-      // TODO(go/non-datastore-allocateid): also check that all the objects have IDs
-      checkArgument(
-          !jpaTm().exists(obj), "Entities created in SqlBatchWriter must not already exist");
-    }
   }
 }

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -14,9 +14,7 @@
 
 package google.registry.beam.common;
 
-import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
-import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
@@ -29,9 +27,9 @@ import org.apache.beam.sdk.options.Description;
  * Defines Nomulus-specific pipeline options, e.g. JPA configurations.
  *
  * <p>When using the Cloud Dataflow runner, users are recommended to set an upper bound on active
- * database connections by setting the pipeline worker options including {@code --maxNumWorkers},
- * {@code workerMachineType}, and {@code numberOfWorkerHarnessThreads}. Please refer to {@link
- * Write#shards()} for more information.
+ * database connections by setting the max number of pipeline worker threads using {@code
+ * --maxNumWorkers} and {@code workerMachineType} for batch pipelines, or {@code --maxNumWorkers}
+ * and {@code --numberOfWorkerHarnessThreads} for streaming pipelines.
  */
 public interface RegistryPipelineOptions extends GcpOptions {
 
@@ -58,25 +56,6 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setSqlWriteBatchSize(int sqlWriteBatchSize);
 
-  @Description(
-      "Number of shards to create out of the data before writing to the SQL database. Please refer "
-          + "to the Javadoc of RegistryJpaIO.Write.shards() for how to choose this value.")
-  @Default.Integer(100)
-  int getSqlWriteShards();
-
-  void setSqlWriteShards(int maxConcurrentSqlWriters);
-
-  @DeleteAfterMigration
-  @Description(
-      "Whether to use self allocated primary IDs when building entities. This should only be used"
-          + " when the IDs are not significant and the resulting entities are not persisted back to"
-          + " the database. Use with caution as self allocated IDs are not unique across workers,"
-          + " and persisting entities with these IDs can be dangerous.")
-  @Default.Boolean(false)
-  boolean getUseSelfAllocatedId();
-
-  void setUseSelfAllocatedId(boolean useSelfAllocatedId);
-
   static RegistryPipelineComponent toRegistryPipelineComponent(RegistryPipelineOptions options) {
     return DaggerRegistryPipelineComponent.builder()
         .isolationOverride(options.getIsolationOverride())

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -21,8 +21,6 @@ import com.google.common.flogger.FluentLogger;
 import dagger.Lazy;
 import google.registry.config.RegistryEnvironment;
 import google.registry.config.SystemPropertySetter;
-import google.registry.model.AppEngineEnvironment;
-import google.registry.model.IdService;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import org.apache.beam.sdk.harness.JvmInitializer;
@@ -63,20 +61,6 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
         transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
     }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
-    // Masquerade all threads as App Engine threads, so we can create Ofy keys in the pipeline. Also
-    // loads all ofy entities.
-    new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
-        .setEnvironmentForAllThreads();
     SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");
-    // Use self-allocated IDs if requested. Note that this inevitably results in duplicate IDs from
-    // multiple workers, which can also collide with existing IDs in the database. So they cannot be
-    // dependent upon for comparison or anything significant. The resulting entities can never be
-    // persisted back into the database. This is a stop-gap measure that should only be used when
-    // you need to create Buildables in Beam, but do not have control over how the IDs are
-    // allocated, and you don't care about the generated IDs as long
-    // as you can build the entities.
-    if (registryOptions.getUseSelfAllocatedId()) {
-      IdService.setForceUseSelfAllocatedId();
-    }
   }
 }

core/src/main/java/google/registry/beam/common/RegistryQuery.java

@@ -14,7 +14,7 @@
 
 package google.registry.beam.common;
 
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import google.registry.persistence.transaction.JpaTransactionManager;
 import java.io.Serializable;
@@ -24,8 +24,10 @@ import java.util.stream.Stream;
 import javax.annotation.Nullable;
 import javax.persistence.EntityManager;
 import javax.persistence.Query;
+import javax.persistence.TemporalType;
 import javax.persistence.TypedQuery;
 import javax.persistence.criteria.CriteriaQuery;
+import org.joda.time.DateTime;
 
 /** Interface for query instances used by {@link RegistryJpaIO.Read}. */
 public interface RegistryQuery<T> extends Serializable {
@@ -53,11 +55,18 @@ public interface RegistryQuery<T> extends Serializable {
   static <T> RegistryQuery<T> createQuery(
       String sql, @Nullable Map<String, Object> parameters, boolean nativeQuery) {
     return () -> {
-      EntityManager entityManager = jpaTm().getEntityManager();
+      EntityManager entityManager = tm().getEntityManager();
       Query query =
           nativeQuery ? entityManager.createNativeQuery(sql) : entityManager.createQuery(sql);
       if (parameters != null) {
-        parameters.forEach(query::setParameter);
+        parameters.forEach(
+            (key, value) -> {
+              if (value instanceof DateTime) {
+                query.setParameter(key, ((DateTime) value).toDate(), TemporalType.TIMESTAMP);
+              } else {
+                query.setParameter(key, value);
+              }
+            });
       }
       JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
       @SuppressWarnings("unchecked")
@@ -76,7 +85,7 @@ public interface RegistryQuery<T> extends Serializable {
       String jpql, @Nullable Map<String, Object> parameters, Class<T> clazz) {
     return () -> {
       // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
-      EntityManager entityManager = jpaTm().getEntityManager();
+      EntityManager entityManager = tm().getEntityManager();
       TypedQuery<T> query = entityManager.createQuery(jpql, clazz);
       if (parameters != null) {
         parameters.forEach(query::setParameter);
@@ -98,7 +107,7 @@ public interface RegistryQuery<T> extends Serializable {
   static <T> RegistryQuery<T> createQuery(CriteriaQuerySupplier<T> criteriaQuery) {
     return () -> {
       // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
-      EntityManager entityManager = jpaTm().getEntityManager();
+      EntityManager entityManager = tm().getEntityManager();
       TypedQuery<T> query = entityManager.createQuery(criteriaQuery.get());
       JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
       return query.getResultStream().map(e -> detach(entityManager, e));

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -26,13 +26,14 @@ import com.google.auto.value.AutoValue;
 import com.google.cloud.storage.BlobId;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
+import google.registry.batch.CloudTasksUtils;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.PgpHelper;
 import google.registry.model.common.Cursor;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.model.rde.RdeRevision;
-import google.registry.model.tld.Registry;
+import google.registry.model.tld.Tld;
 import google.registry.rde.BrdaCopyAction;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.Ghostryde;
@@ -46,7 +47,6 @@ import google.registry.rde.RdeUtil;
 import google.registry.request.Action.Service;
 import google.registry.request.RequestParameters;
 import google.registry.tldconfig.idn.IdnTableEnum;
-import google.registry.util.CloudTasksUtils;
 import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
 import google.registry.xml.ValidationMode;
@@ -272,12 +272,12 @@ public class RdeIO {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
-                Registry registry = Registry.get(key.tld());
+                Tld tld = Tld.get(key.tld());
                 Optional<Cursor> cursor =
                     tm().transact(
                             () ->
                                 tm().loadByKeyIfPresent(
-                                        Cursor.createScopedVKey(key.cursor(), registry)));
+                                        Cursor.createScopedVKey(key.cursor(), tld)));
                 DateTime position = getCursorTimeOrStartOfTime(cursor);
                 checkState(key.interval() != null, "Interval must be present");
                 DateTime newPosition = key.watermark().plus(key.interval());
@@ -290,7 +290,7 @@ public class RdeIO {
                     "Partial ordering of RDE deposits broken: %s %s",
                     position,
                     key);
-                tm().put(Cursor.createScoped(key.cursor(), newPosition, registry));
+                tm().put(Cursor.createScoped(key.cursor(), newPosition, tld));
                 logger.atInfo().log(
                     "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), input.getValue());
@@ -306,7 +306,7 @@ public class RdeIO {
                       RDE_UPLOAD_QUEUE,
                       cloudTasksUtils.createPostTaskWithDelay(
                           RdeUploadAction.PATH,
-                          Service.BACKEND.getServiceId(),
+                          Service.BACKEND,
                           ImmutableMultimap.of(
                               RequestParameters.PARAM_TLD,
                               key.tld(),
@@ -318,7 +318,7 @@ public class RdeIO {
                       BRDA_QUEUE,
                       cloudTasksUtils.createPostTaskWithDelay(
                           BrdaCopyAction.PATH,
-                          Service.BACKEND.getServiceId(),
+                          Service.BACKEND,
                           ImmutableMultimap.of(
                               RequestParameters.PARAM_TLD,
                               key.tld(),

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -26,7 +26,7 @@ import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_HOSTS;
 import static google.registry.beam.rde.RdePipeline.TupleTags.REVISION_ID;
 import static google.registry.beam.rde.RdePipeline.TupleTags.SUPERORDINATE_DOMAINS;
 import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 
 import com.google.common.collect.ImmutableList;
@@ -38,6 +38,7 @@ import com.google.common.flogger.FluentLogger;
 import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
+import google.registry.batch.CloudTasksUtils;
 import google.registry.beam.common.RegistryJpaIO;
 import google.registry.beam.common.RegistryPipelineOptions;
 import google.registry.config.CloudTasksUtilsModule;
@@ -62,7 +63,6 @@ import google.registry.rde.DepositFragment;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
 import google.registry.rde.RdeMarshaller;
-import google.registry.util.CloudTasksUtils;
 import google.registry.util.UtilsModule;
 import google.registry.xml.ValidationMode;
 import java.io.ByteArrayInputStream;
@@ -170,7 +170,6 @@ import org.joda.time.DateTime;
  * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
  *     Flex Templates</a>
  */
-@SuppressWarnings("ALL")
 @Singleton
 public class RdePipeline implements Serializable {
 
@@ -305,7 +304,7 @@ public class RdePipeline implements Serializable {
                     (String registrarRepoId) -> {
                       VKey<Registrar> key = VKey.create(Registrar.class, registrarRepoId);
                       includedRegistrarCounter.inc();
-                      Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
+                      Registrar registrar = tm().transact(() -> tm().loadByKey(key));
                       DepositFragment fragment = marshaller.marshalRegistrar(registrar);
                       ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
                           pendingDeposits.stream()
@@ -376,11 +375,9 @@ public class RdePipeline implements Serializable {
   private <T extends HistoryEntry> EppResource loadResourceByHistoryEntryId(
       Class<T> historyEntryClazz, String repoId, long revisionId) {
 
-    return jpaTm()
-        .transact(
+    return tm().transact(
             () ->
-                jpaTm()
-                    .loadByKey(
+                tm().loadByKey(
                         VKey.create(historyEntryClazz, new HistoryEntryId(repoId, revisionId))))
         .getResourceAtPointInTime()
         .map(resource -> resource.cloneProjectedAtTime(watermark))
@@ -690,13 +687,6 @@ public class RdePipeline implements Serializable {
     RdePipelineOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
 
-    // We need to self allocate the IDs because the pipeline creates EPP resources from history
-    // entries and projects them to watermark. These buildable entities would otherwise request an
-    // ID from datastore, which Beam does not have access to. The IDs are not included in the
-    // deposits or are these entities persisted back to the database, so it is OK to use a self
-    // allocated ID to get around the limitations of beam.
-    options.setUseSelfAllocatedId(true);
-
     RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();

core/src/main/java/google/registry/beam/resave/ResaveAllEppResourcesPipeline.java

@@ -15,7 +15,7 @@
 package google.registry.beam.resave;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.apache.beam.sdk.values.TypeDescriptors.integers;
 
 import com.google.common.collect.ImmutableList;
@@ -33,7 +33,6 @@ import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.VKey;
 import google.registry.util.DateTimeUtils;
 import java.io.Serializable;
-import java.util.concurrent.ThreadLocalRandom;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -143,15 +142,13 @@ public class ResaveAllEppResourcesPipeline implements Serializable {
   /** Projects and re-saves all resources with repo IDs provided by the {@link Read}. */
   private <T extends EppResource> void projectAndResaveResources(
       Pipeline pipeline, Class<T> clazz, Read<?, String> repoIdRead) {
-    int numShards = options.getSqlWriteShards();
     int batchSize = options.getSqlWriteBatchSize();
     String className = clazz.getSimpleName();
     pipeline
         .apply("Read " + className, repoIdRead)
         .apply(
             "Shard data for class" + className,
-            WithKeys.<Integer, String>of(e -> ThreadLocalRandom.current().nextInt(numShards))
-                .withKeyType(integers()))
+            WithKeys.<Integer, String>of(0).withKeyType(integers()))
         .apply(
             "Group into batches for class" + className,
             GroupIntoBatches.<Integer, String>ofSize(batchSize).withShardedKey())
@@ -174,19 +171,18 @@ public class ResaveAllEppResourcesPipeline implements Serializable {
     public void processElement(
         @Element KV<ShardedKey<Integer>, Iterable<String>> element,
         OutputReceiver<Void> outputReceiver) {
-      jpaTm()
-          .transact(
+      tm().transact(
               () -> {
-                DateTime now = jpaTm().getTransactionTime();
+                DateTime now = tm().getTransactionTime();
                 ImmutableList<VKey<? extends EppResource>> keys =
                     Streams.stream(element.getValue())
                         .map(repoId -> VKey.create(clazz, repoId))
                         .collect(toImmutableList());
                 ImmutableList<EppResource> mappedResources =
-                    jpaTm().loadByKeys(keys).values().stream()
+                    tm().loadByKeys(keys).values().stream()
                         .map(r -> r.cloneProjectedAtTime(now))
                         .collect(toImmutableList());
-                jpaTm().putAll(mappedResources);
+                tm().putAll(mappedResources);
               });
     }
   }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -15,7 +15,7 @@
 package google.registry.beam.spec11;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
@@ -45,6 +45,7 @@ import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
@@ -131,9 +132,8 @@ public class Spec11Pipeline implements Serializable {
                   public void processElement(
                       @Element KV<String, String> input, OutputReceiver<DomainNameInfo> output) {
                     Domain domain =
-                        jpaTm()
-                            .transact(
-                                () -> jpaTm().loadByKey(VKey.create(Domain.class, input.getKey())));
+                        tm().transact(
+                                () -> tm().loadByKey(VKey.create(Domain.class, input.getKey())));
                     String emailAddress = input.getValue();
                     if (emailAddress == null) {
                       emailAddress = "";
@@ -155,26 +155,36 @@ public class Spec11Pipeline implements Serializable {
 
   static void saveToSql(
       PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
-    String transformId = "Spec11 Threat Matches";
     LocalDate date = LocalDate.parse(options.getDate(), ISODateTimeFormat.date());
-    threatMatches.apply(
-        "Write to Sql: " + transformId,
-        RegistryJpaIO.<KV<DomainNameInfo, ThreatMatch>>write()
-            .withName(transformId)
-            .withBatchSize(options.getSqlWriteBatchSize())
-            .withShards(options.getSqlWriteShards())
-            .withJpaConverter(
-                (kv) -> {
-                  DomainNameInfo domainNameInfo = kv.getKey();
-                  return new Spec11ThreatMatch.Builder()
-                      .setThreatTypes(
-                          ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
-                      .setCheckDate(date)
-                      .setDomainName(domainNameInfo.domainName())
-                      .setDomainRepoId(domainNameInfo.domainRepoId())
-                      .setRegistrarId(domainNameInfo.registrarId())
-                      .build();
-                }));
+    String transformId = "Spec11 Threat Matches";
+    threatMatches
+        .apply(
+            "Construct objects",
+            ParDo.of(
+                new DoFn<KV<DomainNameInfo, ThreatMatch>, Spec11ThreatMatch>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<DomainNameInfo, ThreatMatch> input,
+                      OutputReceiver<Spec11ThreatMatch> output) {
+                    Spec11ThreatMatch spec11ThreatMatch =
+                        new Spec11ThreatMatch.Builder()
+                            .setThreatTypes(
+                                ImmutableSet.of(ThreatType.valueOf(input.getValue().threatType())))
+                            .setCheckDate(date)
+                            .setDomainName(input.getKey().domainName())
+                            .setDomainRepoId(input.getKey().domainRepoId())
+                            .setRegistrarId(input.getKey().registrarId())
+                            // TODO(b/264416932) Assign id to prevent duplicate inserts.
+                            .build();
+                    output.output(spec11ThreatMatch);
+                  }
+                }))
+        .apply("Prevent Fusing", Reshuffle.viaRandomKey())
+        .apply(
+            "Write to Sql: " + transformId,
+            RegistryJpaIO.<Spec11ThreatMatch>write()
+                .withName(transformId)
+                .withBatchSize(options.getSqlWriteBatchSize()));
   }
 
   static void saveToGcs(

core/src/main/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipeline.java

@@ -0,0 +1,166 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.wipeout;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static org.apache.beam.sdk.values.TypeDescriptors.voids;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.reporting.HistoryEntry.HistoryEntryId;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.VKey;
+import java.io.Serializable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+
+/**
+ * Definition of a Dataflow Flex pipeline template, which finds out {@link ContactHistory} entries
+ * that are older than a given age (excluding the most recent one, even if it falls with the range)
+ * and wipe out PII information in them.
+ *
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha \
+ * --pipeline=wipeOutContactHistoryPii}.
+ *
+ * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
+ */
+public class WipeOutContactHistoryPiiPipeline implements Serializable {
+
+  private static final long serialVersionUID = -4111052675715913820L;
+  private static final TupleTag<Long> REVISIONS_TO_WIPE = new TupleTag<>();
+  private static final TupleTag<Long> MOST_RECENT_REVISION = new TupleTag<>();
+
+  private final DateTime cutoffTime;
+  private final boolean dryRun;
+  private final Counter contactsInScope =
+      Metrics.counter("WipeOutContactHistoryPii", "contacts in scope");
+  private final Counter historiesToWipe =
+      Metrics.counter("WipeOutContactHistoryPii", "contact histories to wipe PII from");
+  private final Counter historiesWiped =
+      Metrics.counter("WipeOutContactHistoryPii", "contact histories actually updated");
+
+  WipeOutContactHistoryPiiPipeline(WipeOutContactHistoryPiiPipelineOptions options) {
+    dryRun = options.getIsDryRun();
+    cutoffTime = DateTime.parse(options.getCutoffTime());
+  }
+
+  void setup(Pipeline pipeline) {
+    KeyedPCollectionTuple.of(REVISIONS_TO_WIPE, getHistoryEntriesToWipe(pipeline))
+        .and(MOST_RECENT_REVISION, getMostRecentHistoryEntries(pipeline))
+        .apply("Group by contact", CoGroupByKey.create())
+        .apply(
+            "Wipe out PII",
+            MapElements.into(voids())
+                .via(
+                    kv -> {
+                      String repoId = kv.getKey();
+                      long mostRecentRevision = kv.getValue().getOnly(MOST_RECENT_REVISION);
+                      ImmutableList<Long> revisionsToWipe =
+                          Streams.stream(kv.getValue().getAll(REVISIONS_TO_WIPE))
+                              .filter(e -> e != mostRecentRevision)
+                              .collect(toImmutableList());
+                      if (revisionsToWipe.isEmpty()) {
+                        return null;
+                      }
+                      contactsInScope.inc();
+                      tm().transact(
+                              () -> {
+                                for (long revisionId : revisionsToWipe) {
+                                  historiesToWipe.inc();
+                                  ContactHistory history =
+                                      tm().loadByKey(
+                                              VKey.create(
+                                                  ContactHistory.class,
+                                                  new HistoryEntryId(repoId, revisionId)));
+                                  // In the unlikely case where multiple pipelines run at the
+                                  // same time, or where the runner decides to rerun a particular
+                                  // transform, we might have a history entry that has already been
+                                  // wiped at this point. There's no need to wipe it again.
+                                  if (!dryRun
+                                      && history.getContactBase().isPresent()
+                                      && history.getContactBase().get().getEmailAddress() != null) {
+                                    historiesWiped.inc();
+                                    tm().update(history.asBuilder().wipeOutPii().build());
+                                  }
+                                }
+                              });
+                      return null;
+                    }));
+  }
+
+  PCollection<KV<String, Long>> getHistoryEntriesToWipe(Pipeline pipeline) {
+    return pipeline.apply(
+        "Find contact histories to wipee",
+        // Email is one of the required fields in EPP, meaning it's initially not null when it
+        // is set by EPP flows (even though it is nullalbe in the SQL schema). Therefore,
+        // checking if it's null is one way to avoid processing contact history entities that
+        // have been processed previously. Refer to RFC 5733 for more information.
+        RegistryJpaIO.read(
+                "SELECT repoId, revisionId FROM ContactHistory WHERE email IS NOT NULL AND"
+                    + " modificationTime < :cutoffTime",
+                ImmutableMap.of("cutoffTime", cutoffTime),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1]))
+            .withCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of())));
+  }
+
+  PCollection<KV<String, Long>> getMostRecentHistoryEntries(Pipeline pipeline) {
+    return pipeline.apply(
+        "Find the most recent historiy entry for each contact",
+        RegistryJpaIO.read(
+                "SELECT repoId, revisionId FROM ContactHistory"
+                    + " WHERE (repoId, modificationTime) IN"
+                    + " (SELECT repoId, MAX(modificationTime) FROM ContactHistory GROUP BY repoId)",
+                ImmutableMap.of(),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1]))
+            .withCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of())));
+  }
+
+  PipelineResult run(Pipeline pipeline) {
+    setup(pipeline);
+    return pipeline.run();
+  }
+
+  public static void main(String[] args) {
+    PipelineOptionsFactory.register(WipeOutContactHistoryPiiPipelineOptions.class);
+    WipeOutContactHistoryPiiPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args)
+            .withValidation()
+            .as(WipeOutContactHistoryPiiPipelineOptions.class);
+    // Repeatable read should be more than enough since we are dealing with old history entries that
+    // are otherwise immutable.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    Pipeline pipeline = Pipeline.create(options);
+    new WipeOutContactHistoryPiiPipeline(options).run(pipeline);
+  }
+}

core/src/main/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipelineOptions.java

@@ -0,0 +1,37 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.wipeout;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+
+public interface WipeOutContactHistoryPiiPipelineOptions extends RegistryPipelineOptions {
+
+  @Description(
+      "A contact history entry with a history modification time before this time will have its PII"
+          + " wiped, unless it is the most entry for the contact.")
+  String getCutoffTime();
+
+  void setCutoffTime(String value);
+
+  @Description(
+      "If true, the wiped out billing events will not be saved but the pipeline metrics counter"
+          + " will still be updated.")
+  @Default.Boolean(false)
+  boolean getIsDryRun();
+
+  void setIsDryRun(boolean value);
+}

core/src/main/java/google/registry/bigquery/BigqueryConnection.java

@@ -36,7 +36,6 @@ import com.google.api.services.bigquery.model.GetQueryResultsResponse;
 import com.google.api.services.bigquery.model.Job;
 import com.google.api.services.bigquery.model.JobConfiguration;
 import com.google.api.services.bigquery.model.JobConfigurationExtract;
-import com.google.api.services.bigquery.model.JobConfigurationLoad;
 import com.google.api.services.bigquery.model.JobConfigurationQuery;
 import com.google.api.services.bigquery.model.JobReference;
 import com.google.api.services.bigquery.model.JobStatistics;
@@ -57,7 +56,6 @@ import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import google.registry.bigquery.BigqueryUtils.DestinationFormat;
-import google.registry.bigquery.BigqueryUtils.SourceFormat;
 import google.registry.bigquery.BigqueryUtils.TableType;
 import google.registry.bigquery.BigqueryUtils.WriteDisposition;
 import google.registry.util.NonFinalForTesting;
@@ -375,23 +373,6 @@ public class BigqueryConnection implements AutoCloseable {
     }
   }
 
-  /**
-   * Starts an asynchronous load job to populate the specified destination table with the given
-   * source URIs and source format. Returns a ListenableFuture that holds the same destination table
-   * object on success.
-   */
-  public ListenableFuture<DestinationTable> startLoad(
-      DestinationTable dest, SourceFormat sourceFormat, Iterable<String> sourceUris) {
-    Job job = new Job()
-        .setConfiguration(new JobConfiguration()
-            .setLoad(new JobConfigurationLoad()
-                .setWriteDisposition(dest.getWriteDisposition().toString())
-                .setSourceFormat(sourceFormat.toString())
-                .setSourceUris(ImmutableList.copyOf(sourceUris))
-                .setDestinationTable(dest.getTableReference())));
-    return transform(runJobToCompletion(job, dest), this::updateTable, directExecutor());
-  }
-
   /**
    * Starts an asynchronous query job to populate the specified destination table with the results
    * of the specified query, or if the table is a view, to update the view to reflect that query.

core/src/main/java/google/registry/bigquery/BigqueryUtils.java

@@ -25,18 +25,6 @@ import org.joda.time.format.ISODateTimeFormat;
 /** Utilities related to Bigquery. */
 public class BigqueryUtils {
 
-  /** Bigquery modes for schema fields. */
-  public enum FieldMode {
-    NULLABLE,
-    REQUIRED,
-    REPEATED;
-
-    /** Return the name of the field mode as it should appear in the Bigquery schema. */
-    public String schemaName() {
-      return name();
-    }
-  }
-
   /** Bigquery schema field types. */
   public enum FieldType {
     STRING,
@@ -44,19 +32,7 @@ public class BigqueryUtils {
     FLOAT,
     TIMESTAMP,
     RECORD,
-    BOOLEAN;
-
-    /** Return the name of the field type as it should appear in the Bigquery schema. */
-    public String schemaName() {
-      return name();
-    }
-  }
-
-  /** Source formats for Bigquery load jobs. */
-  public enum SourceFormat {
-    CSV,
-    NEWLINE_DELIMITED_JSON,
-    DATASTORE_BACKUP
+    BOOLEAN
   }
 
   /** Destination formats for Bigquery extract jobs. */

core/src/main/java/google/registry/config/CloudTasksUtilsModule.java

@@ -19,18 +19,15 @@ import com.google.cloud.tasks.v2.CloudTasksClient;
 import com.google.cloud.tasks.v2.CloudTasksSettings;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.batch.CloudTasksUtils;
+import google.registry.batch.CloudTasksUtils.GcpCloudTasksClient;
+import google.registry.batch.CloudTasksUtils.SerializableCloudTasksClient;
 import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.RegistryConfig.Config;
-import google.registry.util.Clock;
-import google.registry.util.CloudTasksUtils;
-import google.registry.util.CloudTasksUtils.GcpCloudTasksClient;
-import google.registry.util.CloudTasksUtils.SerializableCloudTasksClient;
 import google.registry.util.GoogleCredentialsBundle;
-import google.registry.util.Retrier;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.function.Supplier;
-import javax.inject.Singleton;
 
 /**
  * A {@link Module} that provides {@link CloudTasksUtils}.
@@ -41,17 +38,6 @@ import javax.inject.Singleton;
 @Module
 public abstract class CloudTasksUtilsModule {
 
-  @Singleton
-  @Provides
-  public static CloudTasksUtils provideCloudTasksUtils(
-      @Config("projectId") String projectId,
-      @Config("locationId") String locationId,
-      SerializableCloudTasksClient client,
-      Retrier retrier,
-      Clock clock) {
-    return new CloudTasksUtils(retrier, clock, projectId, locationId, client);
-  }
-
   // Provides a supplier instead of using a Dagger @Provider because the latter is not serializable.
   @Provides
   public static Supplier<CloudTasksClient> provideCloudTasksClientSupplier(

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -32,6 +32,8 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.dns.ReadDnsRefreshRequestsAction;
+import google.registry.model.common.DnsRefreshRequest;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.util.YamlUtils;
 import java.lang.annotation.Documented;
@@ -60,8 +62,8 @@ import org.joda.time.Duration;
  *
  * <p>This class does not represent the total configuration of the Nomulus service. It's <b>only
  * meant for settings that need to be configured <i>once</i></b>. Settings which may be subject to
- * change in the future, should instead be retrieved from Datastore. The {@link
- * google.registry.model.tld.Registry Registry} class is one such example of this.
+ * change in the future, should instead be retrieved from the database. The {@link
+ * google.registry.model.tld.Tld Tld} class is one such example of this.
  *
  * <p>Note: Only settings that are actually configurable belong in this file. It's not a catch-all
  * for anything widely used throughout the code base.
@@ -103,13 +105,25 @@ public final class RegistryConfig {
     @Provides
     @Config("projectId")
     public static String provideProjectId(RegistryConfigSettings config) {
-      return config.appEngine.projectId;
+      return config.gcpProject.projectId;
+    }
+
+    @Provides
+    @Config("serviceAccountEmails")
+    public static ImmutableList<String> provideServiceAccountEmails(RegistryConfigSettings config) {
+      return ImmutableList.copyOf(config.gcpProject.serviceAccountEmails);
+    }
+
+    @Provides
+    @Config("projectIdNumber")
+    public static long provideProjectIdNumber(RegistryConfigSettings config) {
+      return config.gcpProject.projectIdNumber;
     }
 
     @Provides
     @Config("locationId")
     public static String provideLocationId(RegistryConfigSettings config) {
-      return config.appEngine.locationId;
+      return config.gcpProject.locationId;
     }
 
     /**
@@ -282,9 +296,9 @@ public final class RegistryConfig {
 
     /**
      * The maximum number of domain and host updates to batch together to send to
-     * PublishDnsUpdatesAction, to avoid exceeding AppEngine's limits.
+     * PublishDnsUpdatesAction, to avoid exceeding HTTP request timeout limits.
      *
-     * @see google.registry.dns.ReadDnsQueueAction
+     * @see google.registry.dns.ReadDnsRefreshRequestsAction
      */
     @Provides
     @Config("dnsTldUpdateBatchSize")
@@ -315,23 +329,30 @@ public final class RegistryConfig {
     }
 
     /**
-     * The requested maximum duration for ReadDnsQueueAction.
+     * The requested maximum duration for {@link ReadDnsRefreshRequestsAction}.
      *
-     * <p>ReadDnsQueueAction reads update tasks from the dns-pull queue. It will continue reading
-     * tasks until either the queue is empty, or this duration has passed.
+     * <p>{@link ReadDnsRefreshRequestsAction} reads refresh requests from {@link DnsRefreshRequest}
+     * It will continue reading requests until either no requests exist that matche the condition,
+     * or this duration has passed.
      *
-     * <p>This time is the maximum duration between the first and last attempt to lease tasks from
-     * the dns-pull queue. The actual running time might be slightly longer, as we process the
-     * tasks.
+     * <p>This time is the maximum duration between the first and last attempt to read requests from
+     * {@link DnsRefreshRequest}. The actual running time might be slightly longer, as we process
+     * the requests.
      *
-     * <p>This value should be less than the cron-job repeat rate for ReadDnsQueueAction, to make
-     * sure we don't have multiple ReadDnsActions leasing tasks simultaneously.
+     * <p>The requests that are read will not be read again by any action until after this period
+     * has passed, so concurrent runs (or runs that are very close to each other) of {@link
+     * ReadDnsRefreshRequestsAction} will not keep reading the same requests with the earliest
+     * request time.
      *
-     * @see google.registry.dns.ReadDnsQueueAction
+     * <p>Still, this value should ideally be less than the cloud scheduler job repeat rate for
+     * {@link ReadDnsRefreshRequestsAction}, to not waste resources on multiple actions running at
+     * the same time.
+     *
+     * <p>see google.registry.dns.ReadDnsRefreshRequestsAction
      */
     @Provides
-    @Config("readDnsQueueActionRuntime")
-    public static Duration provideReadDnsQueueRuntime() {
+    @Config("readDnsRefreshRequestsActionRuntime")
+    public static Duration provideReadDnsRefreshRequestsRuntime() {
       return Duration.standardSeconds(45);
     }
 
@@ -569,9 +590,9 @@ public final class RegistryConfig {
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *
-     * @see google.registry.beam.invoicing.InvoicingPipeline
+     * @see google.registry.beam.billing.InvoicingPipeline
      * @see google.registry.beam.spec11.Spec11Pipeline
-     * @see google.registry.beam.invoicing.InvoicingPipeline
+     * @see google.registry.beam.billing.InvoicingPipeline
      */
     @Provides
     @Config("defaultJobRegion")
@@ -649,7 +670,7 @@ public final class RegistryConfig {
     /**
      * Returns the URL of the GCS bucket we store invoices and detail reports in.
      *
-     * @see google.registry.beam.invoicing.InvoicingPipeline
+     * @see google.registry.beam.billing.InvoicingPipeline
      */
     @Provides
     @Config("billingBucketUrl")
@@ -685,7 +706,7 @@ public final class RegistryConfig {
     /**
      * Returns the file prefix for the invoice CSV file.
      *
-     * @see google.registry.beam.invoicing.InvoicingPipeline
+     * @see google.registry.beam.billing.InvoicingPipeline
      * @see google.registry.reporting.billing.BillingEmailUtils
      */
     @Provides
@@ -918,7 +939,7 @@ public final class RegistryConfig {
      * <p>Note that this uses {@code @Named} instead of {@code @Config} so that it can be used from
      * the low-level util package, which cannot have a dependency on the config package.
      *
-     * @see google.registry.util.CloudTasksUtils
+     * @see google.registry.batch.CloudTasksUtils
      */
     @Provides
     @Named("transientFailureRetries")
@@ -1147,6 +1168,12 @@ public final class RegistryConfig {
       return ImmutableSet.copyOf(config.oAuth.allowedOauthClientIds);
     }
 
+    @Provides
+    @Config("iapClientId")
+    public static Optional<String> provideIapClientId(RegistryConfigSettings config) {
+      return Optional.ofNullable(config.oAuth.iapClientId);
+    }
+
     /**
      * Provides the OAuth scopes required for accessing Google APIs using the default credential.
      */
@@ -1298,12 +1325,6 @@ public final class RegistryConfig {
       return config.contactHistory.minMonthsBeforeWipeOut;
     }
 
-    @Provides
-    @Config("wipeOutQueryBatchSize")
-    public static int provideWipeOutQueryBatchSize(RegistryConfigSettings config) {
-      return config.contactHistory.wipeOutQueryBatchSize;
-    }
-
     @Provides
     @Config("jdbcBatchSize")
     public static int provideHibernateJdbcBatchSize(RegistryConfigSettings config) {
@@ -1311,21 +1332,47 @@ public final class RegistryConfig {
     }
 
     @Provides
-    @Config("packageCreateLimitEmailSubjectText")
-    public static String providePackageCreateLimitEmailSubjectText(RegistryConfigSettings config) {
-      return config.packageMonitoring.packageCreateLimitEmailSubjectText;
+    @Config("packageCreateLimitEmailSubject")
+    public static String providePackageCreateLimitEmailSubject(RegistryConfigSettings config) {
+      return config.packageMonitoring.packageCreateLimitEmailSubject;
     }
 
     @Provides
-    @Config("packageCreateLimitEmailBodyText")
-    public static String providePackageCreateLimitEmailBodyText(RegistryConfigSettings config) {
-      return config.packageMonitoring.packageCreateLimitEmailBodyText;
+    @Config("packageCreateLimitEmailBody")
+    public static String providePackageCreateLimitEmailBody(RegistryConfigSettings config) {
+      return config.packageMonitoring.packageCreateLimitEmailBody;
+    }
+
+    @Provides
+    @Config("packageDomainLimitWarningEmailSubject")
+    public static String providePackageDomainLimitWarningEmailSubject(
+        RegistryConfigSettings config) {
+      return config.packageMonitoring.packageDomainLimitWarningEmailSubject;
+    }
+
+    @Provides
+    @Config("packageDomainLimitWarningEmailBody")
+    public static String providePackageDomainLimitWarningEmailBody(RegistryConfigSettings config) {
+      return config.packageMonitoring.packageDomainLimitWarningEmailBody;
+    }
+
+    @Provides
+    @Config("packageDomainLimitUpgradeEmailSubject")
+    public static String providePackageDomainLimitUpgradeEmailSubject(
+        RegistryConfigSettings config) {
+      return config.packageMonitoring.packageDomainLimitUpgradeEmailSubject;
+    }
+
+    @Provides
+    @Config("packageDomainLimitUpgradeEmailBody")
+    public static String providePackageDomainLimitUpgradeEmailBody(RegistryConfigSettings config) {
+      return config.packageMonitoring.packageDomainLimitUpgradeEmailBody;
     }
   }
 
   /** Returns the App Engine project ID, which is based off the environment name. */
   public static String getProjectId() {
-    return CONFIG_SETTINGS.get().appEngine.projectId;
+    return CONFIG_SETTINGS.get().gcpProject.projectId;
   }
 
   /**
@@ -1338,7 +1385,7 @@ public final class RegistryConfig {
   }
 
   public static boolean areServersLocal() {
-    return CONFIG_SETTINGS.get().appEngine.isLocal;
+    return CONFIG_SETTINGS.get().gcpProject.isLocal;
   }
 
   /**
@@ -1347,7 +1394,7 @@ public final class RegistryConfig {
    * <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
    */
   public static URL getDefaultServer() {
-    return makeUrl(CONFIG_SETTINGS.get().appEngine.defaultServiceUrl);
+    return makeUrl(CONFIG_SETTINGS.get().gcpProject.defaultServiceUrl);
   }
 
   /**
@@ -1356,7 +1403,7 @@ public final class RegistryConfig {
    * <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
    */
   public static URL getBackendServer() {
-    return makeUrl(CONFIG_SETTINGS.get().appEngine.backendServiceUrl);
+    return makeUrl(CONFIG_SETTINGS.get().gcpProject.backendServiceUrl);
   }
 
   /**
@@ -1365,7 +1412,7 @@ public final class RegistryConfig {
    * <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
    */
   public static URL getToolsServer() {
-    return makeUrl(CONFIG_SETTINGS.get().appEngine.toolsServiceUrl);
+    return makeUrl(CONFIG_SETTINGS.get().gcpProject.toolsServiceUrl);
   }
 
   /**
@@ -1374,7 +1421,7 @@ public final class RegistryConfig {
    * <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
    */
   public static URL getPubapiServer() {
-    return makeUrl(CONFIG_SETTINGS.get().appEngine.pubapiServiceUrl);
+    return makeUrl(CONFIG_SETTINGS.get().gcpProject.pubapiServiceUrl);
   }
 
   /** Returns the amount of time a singleton should be cached, before expiring. */
@@ -1450,11 +1497,6 @@ public final class RegistryConfig {
     return CONFIG_SETTINGS.get().registryPolicy.defaultRegistrarWhoisServer;
   }
 
-  /** Returns the base retry duration that gets doubled after each failure within {@code Ofy}. */
-  public static Duration getBaseOfyRetryDuration() {
-    return Duration.millis(CONFIG_SETTINGS.get().datastore.baseOfyRetryMillis);
-  }
-
   /** Returns the default database transaction isolation. */
   public static String getHibernateConnectionIsolation() {
     return CONFIG_SETTINGS.get().hibernate.connectionIsolation;

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -21,12 +21,11 @@ import java.util.Set;
 /** The POJO that YAML config files are deserialized into. */
 public class RegistryConfigSettings {
 
-  public AppEngine appEngine;
+  public GcpProject gcpProject;
   public GSuite gSuite;
   public OAuth oAuth;
   public CredentialOAuth credentialOAuth;
   public RegistryPolicy registryPolicy;
-  public Datastore datastore;
   public Hibernate hibernate;
   public CloudSql cloudSql;
   public CloudDns cloudDns;
@@ -45,15 +44,17 @@ public class RegistryConfigSettings {
   public DnsUpdate dnsUpdate;
   public PackageMonitoring packageMonitoring;
 
-  /** Configuration options that apply to the entire App Engine project. */
-  public static class AppEngine {
+  /** Configuration options that apply to the entire GCP project. */
+  public static class GcpProject {
     public String projectId;
+    public long projectIdNumber;
     public String locationId;
     public boolean isLocal;
     public String defaultServiceUrl;
     public String backendServiceUrl;
     public String toolsServiceUrl;
     public String pubapiServiceUrl;
+    public List<String> serviceAccountEmails;
   }
 
   /** Configuration options for OAuth settings for authenticating users. */
@@ -61,6 +62,7 @@ public class RegistryConfigSettings {
     public List<String> availableOauthScopes;
     public List<String> requiredOauthScopes;
     public List<String> allowedOauthClientIds;
+    public String iapClientId;
   }
 
   /** Configuration options for accessing Google APIs. */
@@ -108,11 +110,6 @@ public class RegistryConfigSettings {
     public boolean requireSslCertificates;
   }
 
-  /** Configuration for Cloud Datastore. */
-  public static class Datastore {
-    public int baseOfyRetryMillis;
-  }
-
   /** Configuration for Hibernate. */
   public static class Hibernate {
     public String connectionIsolation;
@@ -246,7 +243,6 @@ public class RegistryConfigSettings {
   /** Configuration for contact history. */
   public static class ContactHistory {
     public int minMonthsBeforeWipeOut;
-    public int wipeOutQueryBatchSize;
   }
 
   /** Configuration for dns update. */
@@ -260,7 +256,11 @@ public class RegistryConfigSettings {
 
   /** Configuration for package compliance monitoring. */
   public static class PackageMonitoring {
-    public String packageCreateLimitEmailSubjectText;
-    public String packageCreateLimitEmailBodyText;
+    public String packageCreateLimitEmailSubject;
+    public String packageCreateLimitEmailBody;
+    public String packageDomainLimitWarningEmailSubject;
+    public String packageDomainLimitWarningEmailBody;
+    public String packageDomainLimitUpgradeEmailSubject;
+    public String packageDomainLimitUpgradeEmailBody;
   }
 }

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -5,12 +5,14 @@
 # to override some of these values to configure and enable some services used in
 # production environments.
 
-appEngine:
-  # Globally unique App Engine project ID
+gcpProject:
+  # Globally unique GCP project ID
   projectId: registry-project-id
-  # Location of the App engine project, note that us-central1 and europe-west1 are special in that
-  # they are used without the trailing number in App Engine commands and Google Cloud Console.
-  # See: https://cloud.google.com/appengine/docs/locations
+  # Corresponding project ID number
+  projectIdNumber: 123456789012
+  # Location of the GCP project, note that us-central1 and europe-west1 are special in that
+  # they are used without the trailing number in GCP commands and Google Cloud Console.
+  # See: https://cloud.google.com/appengine/docs/locations as an example
   locationId: registry-location-id
 
   # whether to use local/test credentials when connecting to the servers
@@ -20,6 +22,11 @@ appEngine:
   backendServiceUrl: https://localhost
   toolsServiceUrl: https://localhost
   pubapiServiceUrl: https://localhost
+  # Service accounts eligible for authorization (e.g. default service account,
+  # account used by Cloud Scheduler) to send authenticated requests.
+  serviceAccountEmails:
+  - default-service-account-email@email.com
+  - cloud-scheduler-email@email.com
 
 gSuite:
   # Publicly accessible domain name of the running G Suite instance.
@@ -182,11 +189,6 @@ registryPolicy:
   # should generally be true for production environments, for added security.
   requireSslCertificates: true
 
-datastore:
-  # Milliseconds that Objectify waits to retry a Datastore transaction (this
-  # doubles after each failure).
-  baseOfyRetryMillis: 100
-
 hibernate:
   # Make 'SERIALIZABLE' the default isolation level to ensure correctness.
   #
@@ -309,6 +311,9 @@ oAuth:
   # in this list. Client IDs are typically of the format
   # numbers-alphanumerics.apps.googleusercontent.com
   allowedOauthClientIds: []
+  # GCP Identity-Aware Proxy client ID, if set up (note: this requires manual setup
+  # of User objects in the database for Nomulus tool users)
+  iapClientId: null
 
 credentialOAuth:
   # OAuth scopes required for accessing Google APIs using the default
@@ -427,7 +432,7 @@ misc:
 
 beam:
   # The default region to run Apache Beam (Cloud Dataflow) jobs in.
-  defaultJobRegion: us-east1
+  defaultJobRegion: us-central1
   # The GCE machine type to use when a job is CPU-intensive (e. g. RDE). Be sure
   # to check the VM CPU quota for the job region. In a massively parallel
   # pipeline this quota can be easily reached and needs to be raised, otherwise
@@ -469,8 +474,6 @@ registryTool:
 contactHistory:
   # The number of months that a ContactHistory entity should be stored in the database.
   minMonthsBeforeWipeOut: 18
-  # The batch size for querying ContactHistory table in the database.
-  wipeOutQueryBatchSize: 500
 
 # Configuration options relevant to the DNS update functionality.
 dnsUpdate:
@@ -538,18 +541,53 @@ sslCertificateValidation:
 
 # Configuration options for the package compliance monitoring
 packageMonitoring:
-  # Email subject text to notify partners their package has exceeded the limit for domain creates
-  packageCreateLimitEmailSubjectText: "NOTICE: Your Package Is Being Upgraded"
-  # Email body text template notify partners their package has exceeded the limit for domain creates
-  packageCreateLimitEmailBodyText: >
-    Dear %1$s,
+  # Email subject text to notify tech support that a package has exceeded the limit for domain creates
+  packageCreateLimitEmailSubject: "ACTION REQUIRED: Package needs to be upgraded"
+  # Email body text template notify support that a package has exceeded the limit for domain creates
+  packageCreateLimitEmailBody: >
+    Dear Support,
     
-    We are contacting you to inform you that your package with the package token 
-    %2$s has exceeded its limit for annual domain creations.
-    Your package will now be upgraded to the next tier. 
+    A package has exceeded its max create limit and needs to be upgraded to the 
+    next tier.
     
-    If you have any questions or require additional support, please contact us
-    at %3$s.
+    Package ID: %1$s
+    Package Token: %2$s
+    Registrar: %3$s
+    Current Max Create Limit: %4$s
+    Creates Completed: %5$s
+
+  # Email subject text to notify support that a package has exceeded the limit
+  # for current active domains and a warning needs to be sent
+  packageDomainLimitWarningEmailSubject: "ACTION REQUIRED: Package has exceeded the domain limit - send warning"
+  # Email body text template to inform support that a package has exceeded the
+  # limit for active domains and a warning needs to be sent that the package
+  # will be upgraded in 30 days
+  packageDomainLimitWarningEmailBody: >
+    Dear Support,
     
-    Regards,
-    Example Registry
+    A package has exceeded its max domain limit. Please send a warning to the 
+    registrar that their package will be upgraded to the next tier in 30 days if 
+    the number of active domains does not return below the limit.
+    
+    Package ID: %1$s
+    Package Token: %2$s
+    Registrar: %3$s
+    Active Domain Limit: %4$s
+    Current Active Domains: %5$s
+
+  # Email subject text to notify support that a package has exceeded the limit
+  # for current active domains for more than 30 days and needs to be upgraded
+  packageDomainLimitUpgradeEmailSubject: "ACTION REQUIRED: Package has exceeded the domain limit - upgrade package"
+  # Email body text template to inform support that a package has exceeded the
+  # limit for active domains for more than 30 days and needs to be upgraded
+  packageDomainLimitUpgradeEmailBody: >
+    Dear Support,
+    
+    A package has exceeded its max domain limit for over 30 days and needs to be 
+    upgraded to the next tier.
+    
+    Package ID: %1$s
+    Package Token: %2$s
+    Registrar: %3$s
+    Active Domain Limit: %4$s
+    Current Active Domains: %5$s

core/src/main/java/google/registry/config/files/nomulus-config-production-sample.yaml

@@ -2,7 +2,7 @@
 # This is the same as what Google Registry runs in production, except with
 # placeholders for Google-specific settings.
 
-appEngine:
+gcpProject:
   projectId: placeholder
   # Set to true if running against local servers (localhost)
   isLocal: false

core/src/main/java/google/registry/config/files/nomulus-config-unittest.yaml

@@ -10,10 +10,6 @@ registryPolicy:
     Disclaimer line 1.
     Line 2 is this 1.
 
-datastore:
-  eppResourceIndexBucketsNum: 3
-  baseOfyRetryMillis: 0
-
 caching:
   singletonCacheRefreshSeconds: 0
   domainLabelCachingSeconds: 0

core/src/main/java/google/registry/cron/TldFanoutAction.java

@@ -28,8 +28,8 @@ import static google.registry.cron.CronModule.JITTER_SECONDS_PARAM;
 import static google.registry.cron.CronModule.QUEUE_PARAM;
 import static google.registry.cron.CronModule.RUN_IN_EMPTY_PARAM;
 import static google.registry.model.tld.Registries.getTldsOfType;
-import static google.registry.model.tld.Registry.TldType.REAL;
-import static google.registry.model.tld.Registry.TldType.TEST;
+import static google.registry.model.tld.Tld.TldType.REAL;
+import static google.registry.model.tld.Tld.TldType.TEST;
 
 import com.google.cloud.tasks.v2.Task;
 import com.google.common.collect.ArrayListMultimap;
@@ -38,6 +38,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
+import google.registry.batch.CloudTasksUtils;
 import google.registry.request.Action;
 import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
@@ -45,7 +46,6 @@ import google.registry.request.ParameterMap;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
 import java.util.stream.Stream;
 import javax.inject.Inject;
@@ -158,6 +158,6 @@ public final class TldFanoutAction implements Runnable {
       params.put(RequestParameters.PARAM_TLD, tld);
     }
     return cloudTasksUtils.createPostTaskWithJitter(
-        endpoint, Service.BACKEND.toString(), params, jitterSeconds);
+        endpoint, Service.BACKEND, params, jitterSeconds);
   }
 }

core/src/main/java/google/registry/dns/DnsConstants.java

@@ -1,38 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.dns;
-
-/** Static class for DNS-related constants. */
-public class DnsConstants {
-  private DnsConstants() {}
-
-  /** The name of the DNS pull queue. */
-  public static final String DNS_PULL_QUEUE_NAME = "dns-pull";  // See queue.xml.
-
-  /** The name of the DNS publish push queue. */
-  public static final String DNS_PUBLISH_PUSH_QUEUE_NAME = "dns-publish";  // See queue.xml.
-
-  /** The parameter to use for storing the target type ("domain" or "host" or "zone"). */
-  public static final String DNS_TARGET_TYPE_PARAM = "Target-Type";
-
-  /** The parameter to use for storing the target name (domain or host name) with the task. */
-  public static final String DNS_TARGET_NAME_PARAM = "Target-Name";
-
-  /** The parameter to use for storing the creation time with the task. */
-  public static final String DNS_TARGET_CREATE_TIME_PARAM = "Create-Time";
-
-  /** The possible values of the {@code DNS_TARGET_TYPE_PARAM} parameter. */
-  public enum TargetType { DOMAIN, HOST, ZONE }
-}

core/src/main/java/google/registry/dns/DnsModule.java

@@ -14,27 +14,25 @@
 
 package google.registry.dns;
 
-import static google.registry.dns.DnsConstants.DNS_PUBLISH_PUSH_QUEUE_NAME;
-import static google.registry.dns.DnsConstants.DNS_PULL_QUEUE_NAME;
 import static google.registry.dns.RefreshDnsOnHostRenameAction.PARAM_HOST_KEY;
 import static google.registry.request.RequestParameters.extractEnumParameter;
 import static google.registry.request.RequestParameters.extractIntParameter;
+import static google.registry.request.RequestParameters.extractOptionalIntParameter;
+import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 import static google.registry.request.RequestParameters.extractSetOfParameters;
 
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.QueueFactory;
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
 import dagger.Binds;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.dns.DnsConstants.TargetType;
+import google.registry.dns.DnsUtils.TargetType;
 import google.registry.dns.writer.DnsWriterZone;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
+import java.util.Optional;
 import java.util.Set;
-import javax.inject.Named;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.DateTime;
 
@@ -48,7 +46,10 @@ public abstract class DnsModule {
   public static final String PARAM_DOMAINS = "domains";
   public static final String PARAM_HOSTS = "hosts";
   public static final String PARAM_PUBLISH_TASK_ENQUEUED = "enqueued";
-  public static final String PARAM_REFRESH_REQUEST_CREATED = "itemsCreated";
+  public static final String PARAM_REFRESH_REQUEST_TIME = "requestTime";
+  // This parameter cannot be named "jitterSeconds", which will be read by TldFanoutAction and not
+  // be passed down to actions.
+  public static final String PARAM_DNS_JITTER_SECONDS = "dnsJitterSeconds";
 
   @Binds
   @DnsWriterZone
@@ -65,28 +66,19 @@ public abstract class DnsModule {
     return Hashing.murmur3_32_fixed();
   }
 
-  @Provides
-  @Named(DNS_PULL_QUEUE_NAME)
-  static Queue provideDnsPullQueue() {
-    return QueueFactory.getQueue(DNS_PULL_QUEUE_NAME);
-  }
-
-  @Provides
-  @Named(DNS_PUBLISH_PUSH_QUEUE_NAME)
-  static Queue provideDnsUpdatePushQueue() {
-    return QueueFactory.getQueue(DNS_PUBLISH_PUSH_QUEUE_NAME);
-  }
-
   @Provides
   @Parameter(PARAM_PUBLISH_TASK_ENQUEUED)
   static DateTime provideCreateTime(HttpServletRequest req) {
     return DateTime.parse(extractRequiredParameter(req, PARAM_PUBLISH_TASK_ENQUEUED));
   }
 
+  // TODO: Retire the old header after DNS pull queue migration.
   @Provides
-  @Parameter(PARAM_REFRESH_REQUEST_CREATED)
+  @Parameter(PARAM_REFRESH_REQUEST_TIME)
   static DateTime provideItemsCreateTime(HttpServletRequest req) {
-    return DateTime.parse(extractRequiredParameter(req, PARAM_REFRESH_REQUEST_CREATED));
+    return DateTime.parse(
+        extractOptionalParameter(req, "itemsCreated")
+            .orElse(extractRequiredParameter(req, PARAM_REFRESH_REQUEST_TIME)));
   }
 
   @Provides
@@ -125,6 +117,12 @@ public abstract class DnsModule {
     return extractRequiredParameter(req, PARAM_HOST_KEY);
   }
 
+  @Provides
+  @Parameter(PARAM_DNS_JITTER_SECONDS)
+  static Optional<Integer> provideJitterSeconds(HttpServletRequest req) {
+    return extractOptionalIntParameter(req, PARAM_DNS_JITTER_SECONDS);
+  }
+
   @Provides
   @Parameter("domainOrHostName")
   static String provideName(HttpServletRequest req) {

core/src/main/java/google/registry/dns/DnsQueue.java

@@ -1,170 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.dns;
-
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.common.base.Preconditions.checkArgument;
-import static google.registry.dns.DnsConstants.DNS_PULL_QUEUE_NAME;
-import static google.registry.dns.DnsConstants.DNS_TARGET_CREATE_TIME_PARAM;
-import static google.registry.dns.DnsConstants.DNS_TARGET_NAME_PARAM;
-import static google.registry.dns.DnsConstants.DNS_TARGET_TYPE_PARAM;
-import static google.registry.model.tld.Registries.assertTldExists;
-import static google.registry.request.RequestParameters.PARAM_TLD;
-import static google.registry.util.DomainNameUtils.getTldFromDomainName;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.QueueConstants;
-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
-import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.apphosting.api.DeadlineExceededException;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.flogger.FluentLogger;
-import com.google.common.net.InternetDomainName;
-import com.google.common.util.concurrent.RateLimiter;
-import google.registry.dns.DnsConstants.TargetType;
-import google.registry.model.tld.Registries;
-import google.registry.util.Clock;
-import google.registry.util.NonFinalForTesting;
-import java.util.List;
-import java.util.Optional;
-import java.util.logging.Level;
-import javax.inject.Inject;
-import javax.inject.Named;
-import org.joda.time.Duration;
-
-/**
- * Methods for manipulating the queue used for DNS write tasks.
- *
- * <p>This includes a {@link RateLimiter} to limit the {@link Queue#leaseTasks} call rate to 9 QPS,
- * to stay under the 10 QPS limit for this function.
- *
- * <p>Note that overlapping calls to {@link ReadDnsQueueAction} (the only place where
- * {@link DnsQueue#leaseTasks} is used) will have different rate limiters, so they could exceed the
- * allowed rate. This should be rare though - because {@link DnsQueue#leaseTasks} is only used in
- * {@link ReadDnsQueueAction}, which is run as a cron job with running time shorter than the cron
- * repeat time - meaning there should never be two instances running at once.
- *
- * @see google.registry.config.RegistryConfig.ConfigModule#provideReadDnsQueueRuntime
- */
-public class DnsQueue {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
-  private final Queue queue;
-
-  final Clock clock;
-
-  // Queue.leaseTasks is limited to 10 requests per second as per
-  // https://cloud.google.com/appengine/docs/standard/java/javadoc/com/google/appengine/api/taskqueue/Queue.html
-  // "If you generate more than 10 LeaseTasks requests per second, only the first 10 requests will
-  // return results. The others will return no results."
-  private static final RateLimiter rateLimiter = RateLimiter.create(9);
-
-  @Inject
-  public DnsQueue(@Named(DNS_PULL_QUEUE_NAME) Queue queue, Clock clock) {
-    this.queue = queue;
-    this.clock = clock;
-  }
-
-  @VisibleForTesting
-  public static DnsQueue createForTesting(Clock clock) {
-    return new DnsQueue(getQueue(DNS_PULL_QUEUE_NAME), clock);
-  }
-
-  @NonFinalForTesting
-  @VisibleForTesting
-  long leaseTasksBatchSize = QueueConstants.maxLeaseCount();
-
-  /** Enqueues the given task type with the given target name to the DNS queue. */
-  private TaskHandle addToQueue(
-      TargetType targetType, String targetName, String tld, Duration countdown) {
-    logger.atInfo().log(
-        "Adding task type=%s, target=%s, tld=%s to pull queue %s (%d tasks currently on queue).",
-        targetType, targetName, tld, DNS_PULL_QUEUE_NAME, queue.fetchStatistics().getNumTasks());
-    return queue.add(
-        TaskOptions.Builder.withDefaults()
-            .method(Method.PULL)
-            .countdownMillis(countdown.getMillis())
-            .param(DNS_TARGET_TYPE_PARAM, targetType.toString())
-            .param(DNS_TARGET_NAME_PARAM, targetName)
-            .param(DNS_TARGET_CREATE_TIME_PARAM, clock.nowUtc().toString())
-            .param(PARAM_TLD, tld));
-  }
-
-  /** Adds a task to the queue to refresh the DNS information for the specified subordinate host. */
-  public TaskHandle addHostRefreshTask(String hostName) {
-    Optional<InternetDomainName> tld = Registries.findTldForName(InternetDomainName.from(hostName));
-    checkArgument(
-        tld.isPresent(), String.format("%s is not a subordinate host to a known tld", hostName));
-    return addToQueue(TargetType.HOST, hostName, tld.get().toString(), Duration.ZERO);
-  }
-
-  /** Enqueues a task to refresh DNS for the specified domain now. */
-  public TaskHandle addDomainRefreshTask(String domainName) {
-    return addDomainRefreshTask(domainName, Duration.ZERO);
-  }
-
-  /** Enqueues a task to refresh DNS for the specified domain at some point in the future. */
-  public TaskHandle addDomainRefreshTask(String domainName, Duration countdown) {
-    return addToQueue(
-        TargetType.DOMAIN,
-        domainName,
-        assertTldExists(getTldFromDomainName(domainName)),
-        countdown);
-  }
-
-  /** Adds a task to the queue to refresh the DNS information for the specified zone. */
-  public TaskHandle addZoneRefreshTask(String zoneName) {
-    return addToQueue(TargetType.ZONE, zoneName, zoneName, Duration.ZERO);
-  }
-
-  /**
-   * Returns the maximum number of tasks that can be leased with {@link #leaseTasks}.
-   *
-   * <p>If this many tasks are returned, then there might be more tasks still waiting in the queue.
-   *
-   * <p>If less than this number of tasks are returned, then there are no more items in the queue.
-   */
-  public long getLeaseTasksBatchSize() {
-    return leaseTasksBatchSize;
-  }
-
-  /** Returns handles for a batch of tasks, leased for the specified duration. */
-  public List<TaskHandle> leaseTasks(Duration leaseDuration) {
-    try {
-      rateLimiter.acquire();
-      int numTasks = queue.fetchStatistics().getNumTasks();
-      logger.at((numTasks >= leaseTasksBatchSize) ? Level.WARNING : Level.INFO).log(
-          "There are %d tasks in the DNS queue '%s'.", numTasks, DNS_PULL_QUEUE_NAME);
-      return queue.leaseTasks(leaseDuration.getMillis(), MILLISECONDS, leaseTasksBatchSize);
-    } catch (TransientFailureException | DeadlineExceededException e) {
-      logger.atSevere().withCause(e).log("Failed leasing tasks too fast.");
-      return ImmutableList.of();
-    }
-  }
-
-  /** Delete a list of tasks, removing them from the queue permanently. */
-  public void deleteTasks(List<TaskHandle> tasks) {
-    try {
-      queue.deleteTask(tasks);
-    } catch (TransientFailureException | DeadlineExceededException e) {
-      logger.atSevere().withCause(e).log("Failed deleting tasks too fast.");
-    }
-  }
-}

core/src/main/java/google/registry/dns/DnsUtils.java

@@ -0,0 +1,133 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.dns;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.net.InternetDomainName;
+import google.registry.model.common.DnsRefreshRequest;
+import google.registry.model.tld.Registries;
+import google.registry.model.tld.Tld;
+import java.util.Collection;
+import java.util.Optional;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/** Utility class to handle DNS refresh requests. */
+public final class DnsUtils {
+
+  /** The name of the DNS publish push queue. */
+  public static final String DNS_PUBLISH_PUSH_QUEUE_NAME = "dns-publish"; // See queue.xml.
+
+  private DnsUtils() {}
+
+  private static void requestDnsRefresh(String name, TargetType type, Duration delay) {
+    // Throws an IllegalArgumentException if the name is not under a managed TLD -- we only update
+    // DNS for names that are under our management.
+    String tld = Registries.findTldForNameOrThrow(InternetDomainName.from(name)).toString();
+    tm().transact(
+            () ->
+                tm().insert(
+                        new DnsRefreshRequest(
+                            type, name, tld, tm().getTransactionTime().plus(delay))));
+  }
+
+  public static void requestDomainDnsRefresh(String domainName, Duration delay) {
+    requestDnsRefresh(domainName, TargetType.DOMAIN, delay);
+  }
+
+  public static void requestDomainDnsRefresh(String domainName) {
+    requestDomainDnsRefresh(domainName, Duration.ZERO);
+  }
+
+  public static void requestHostDnsRefresh(String hostName) {
+    requestDnsRefresh(hostName, TargetType.HOST, Duration.ZERO);
+  }
+
+  /**
+   * Returns pending DNS update requests that need further processing up to batch size, in ascending
+   * order of their request time, and updates their processing time to now.
+   *
+   * <p>The criteria to pick the requests to include are:
+   *
+   * <ul>
+   *   <li>They are for the given TLD.
+   *   <li>Their request time is not in the future.
+   *   <li>The last time they were processed is before the cooldown period.
+   * </ul>
+   */
+  public static ImmutableList<DnsRefreshRequest> readAndUpdateRequestsWithLatestProcessTime(
+      String tld, Duration cooldown, int batchSize) {
+    return tm().transact(
+            () -> {
+              DateTime transactionTime = tm().getTransactionTime();
+              ImmutableList<DnsRefreshRequest> requests =
+                  tm().query(
+                          "FROM DnsRefreshRequest WHERE tld = :tld "
+                              + "AND requestTime <= :now AND lastProcessTime < :cutoffTime "
+                              + "ORDER BY requestTime ASC, id ASC",
+                          DnsRefreshRequest.class)
+                      .setParameter("tld", tld)
+                      .setParameter("now", transactionTime)
+                      .setParameter("cutoffTime", transactionTime.minus(cooldown))
+                      .setMaxResults(batchSize)
+                      .getResultStream()
+                      // Note that the process time is when the request was last read, batched and
+                      // queued up for publishing, not when it is actually published by the DNS
+                      // writer. This timestamp acts as a cooldown so the same request will not be
+                      // retried too frequently. See DnsRefreshRequest.getLastProcessTime for a
+                      // detailed explanation.
+                      .map(e -> e.updateProcessTime(transactionTime))
+                      .collect(toImmutableList());
+              tm().updateAll(requests);
+              return requests;
+            });
+  }
+
+  /**
+   * Removes the requests that have been processed.
+   *
+   * <p>Note that if a request entity has already been deleted, the method still succeeds without
+   * error because all we care about is that it no longer exists after the method runs.
+   */
+  public static void deleteRequests(Collection<DnsRefreshRequest> requests) {
+    tm().transact(
+            () ->
+                tm().delete(
+                        requests.stream()
+                            .map(DnsRefreshRequest::createVKey)
+                            .collect(toImmutableList())));
+  }
+
+  public static long getDnsAPlusAAAATtlForHost(String host, Duration dnsDefaultATtl) {
+    Optional<InternetDomainName> tldName = Registries.findTldForName(InternetDomainName.from(host));
+    Duration dnsAPlusAaaaTtl = dnsDefaultATtl;
+    if (tldName.isPresent()) {
+      Tld tld = Tld.get(tldName.get().toString());
+      if (tld.getDnsAPlusAaaaTtl().isPresent()) {
+        dnsAPlusAaaaTtl = tld.getDnsAPlusAaaaTtl().get();
+      }
+    }
+    return dnsAPlusAaaaTtl.getStandardSeconds();
+  }
+
+  /** The possible values of the {@code DNS_TARGET_TYPE_PARAM} parameter. */
+  public enum TargetType {
+    DOMAIN,
+    HOST
+  }
+}

core/src/main/java/google/registry/dns/DnsWriterProxy.java

@@ -19,7 +19,7 @@ import static com.google.common.base.Preconditions.checkState;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.dns.writer.DnsWriter;
-import google.registry.model.tld.Registry;
+import google.registry.model.tld.Tld;
 import java.util.Map;
 import javax.inject.Inject;
 
@@ -41,7 +41,7 @@ public final class DnsWriterProxy {
    * If the DnsWriter doesn't belong to this TLD, will return null.
    */
   public DnsWriter getByClassNameForTld(String className, String tld) {
-    if (!Registry.get(tld).getDnsWriters().contains(className)) {
+    if (!Tld.get(tld).getDnsWriters().contains(className)) {
       logger.atWarning().log(
           "Loaded potentially stale DNS writer %s which is not active on TLD %s.", className, tld);
       return null;

core/src/main/java/google/registry/dns/PublishDnsUpdatesAction.java

@@ -15,14 +15,16 @@
 package google.registry.dns;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.dns.DnsConstants.DNS_PUBLISH_PUSH_QUEUE_NAME;
 import static google.registry.dns.DnsModule.PARAM_DNS_WRITER;
 import static google.registry.dns.DnsModule.PARAM_DOMAINS;
 import static google.registry.dns.DnsModule.PARAM_HOSTS;
 import static google.registry.dns.DnsModule.PARAM_LOCK_INDEX;
 import static google.registry.dns.DnsModule.PARAM_NUM_PUBLISH_LOCKS;
 import static google.registry.dns.DnsModule.PARAM_PUBLISH_TASK_ENQUEUED;
-import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_CREATED;
+import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_TIME;
+import static google.registry.dns.DnsUtils.DNS_PUBLISH_PUSH_QUEUE_NAME;
+import static google.registry.dns.DnsUtils.requestDomainDnsRefresh;
+import static google.registry.dns.DnsUtils.requestHostDnsRefresh;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.request.RequestParameters.PARAM_TLD;
@@ -35,6 +37,7 @@ import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.InternetDomainName;
 import dagger.Lazy;
+import google.registry.batch.CloudTasksUtils;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.dns.DnsMetrics.ActionStatus;
 import google.registry.dns.DnsMetrics.CommitStatus;
@@ -44,7 +47,7 @@ import google.registry.model.domain.Domain;
 import google.registry.model.host.Host;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarPoc;
-import google.registry.model.tld.Registry;
+import google.registry.model.tld.Tld;
 import google.registry.request.Action;
 import google.registry.request.Action.Service;
 import google.registry.request.Header;
@@ -54,7 +57,6 @@ import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.request.lock.LockHandler;
 import google.registry.util.Clock;
-import google.registry.util.CloudTasksUtils;
 import google.registry.util.DomainNameUtils;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
@@ -85,7 +87,6 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private final DnsQueue dnsQueue;
   private final DnsWriterProxy dnsWriterProxy;
   private final DnsMetrics dnsMetrics;
   private final Duration timeout;
@@ -94,9 +95,9 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
   /**
    * The DNS writer to use for this batch.
    *
-   * <p>This comes from the fanout in {@link ReadDnsQueueAction} which dispatches each batch to be
-   * published by each DNS writer on the TLD. So this field contains the value of one of the DNS
-   * writers configured in {@link Registry#getDnsWriters()}, as of the time the batch was written
+   * <p>This comes from the fanout in {@link ReadDnsRefreshRequestsAction} which dispatches each
+   * batch to be published by each DNS writer on the TLD. So this field contains the value of one of
+   * the DNS writers configured in {@link Tld#getDnsWriters()}, as of the time the batch was written
    * out (and not necessarily currently).
    */
   private final String dnsWriter;
@@ -124,7 +125,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
   public PublishDnsUpdatesAction(
       @Parameter(PARAM_DNS_WRITER) String dnsWriter,
       @Parameter(PARAM_PUBLISH_TASK_ENQUEUED) DateTime enqueuedTime,
-      @Parameter(PARAM_REFRESH_REQUEST_CREATED) DateTime itemsCreateTime,
+      @Parameter(PARAM_REFRESH_REQUEST_TIME) DateTime itemsCreateTime,
       @Parameter(PARAM_LOCK_INDEX) int lockIndex,
       @Parameter(PARAM_NUM_PUBLISH_LOCKS) int numPublishLocks,
       @Parameter(PARAM_DOMAINS) Set<String> domains,
@@ -139,7 +140,6 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
       @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress,
       @Header(APP_ENGINE_RETRY_HEADER) Optional<Integer> appEngineRetryCount,
       @Header(CLOUD_TASKS_RETRY_HEADER) Optional<Integer> cloudTasksRetryCount,
-      DnsQueue dnsQueue,
       DnsWriterProxy dnsWriterProxy,
       DnsMetrics dnsMetrics,
       LockHandler lockHandler,
@@ -147,12 +147,11 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
       CloudTasksUtils cloudTasksUtils,
       SendEmailService sendEmailService,
       Response response) {
-    this.dnsQueue = dnsQueue;
     this.dnsWriterProxy = dnsWriterProxy;
     this.dnsMetrics = dnsMetrics;
     this.timeout = timeout;
     this.sendEmailService = sendEmailService;
-    this.retryCount =
+    retryCount =
         cloudTasksRetryCount.orElse(
             appEngineRetryCount.orElseThrow(
                 () -> new IllegalStateException("Missing a valid retry count header")));
@@ -276,7 +275,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     return null;
   }
 
-  private InternetAddress emailToInternetAddress(String email) {
+  private static InternetAddress emailToInternetAddress(String email) {
     try {
       return new InternetAddress(email, true);
     } catch (Exception e) {
@@ -306,7 +305,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
           registrar.get().getContacts().stream()
               .filter(c -> c.getTypes().contains(RegistrarPoc.Type.ADMIN))
               .map(RegistrarPoc::getEmailAddress)
-              .map(this::emailToInternetAddress)
+              .map(PublishDnsUpdatesAction::emailToInternetAddress)
               .collect(toImmutableList());
 
       sendEmailService.sendEmail(
@@ -339,14 +338,14 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
         DNS_PUBLISH_PUSH_QUEUE_NAME,
         cloudTasksUtils.createPostTask(
             PATH,
-            Service.BACKEND.toString(),
+            Service.BACKEND,
             ImmutableMultimap.<String, String>builder()
                 .put(PARAM_TLD, tld)
                 .put(PARAM_DNS_WRITER, dnsWriter)
                 .put(PARAM_LOCK_INDEX, Integer.toString(lockIndex))
                 .put(PARAM_NUM_PUBLISH_LOCKS, Integer.toString(numPublishLocks))
                 .put(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-                .put(PARAM_REFRESH_REQUEST_CREATED, itemsCreateTime.toString())
+                .put(PARAM_REFRESH_REQUEST_TIME, itemsCreateTime.toString())
                 .put(PARAM_DOMAINS, Joiner.on(",").join(domains))
                 .put(PARAM_HOSTS, Joiner.on(",").join(hosts))
                 .build()));
@@ -356,10 +355,10 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
   private void requeueBatch() {
     logger.atInfo().log("Requeueing batch for retry.");
     for (String domain : nullToEmpty(domains)) {
-      dnsQueue.addDomainRefreshTask(domain);
+      requestDomainDnsRefresh(domain);
     }
     for (String host : nullToEmpty(hosts)) {
-      dnsQueue.addHostRefreshTask(host);
+      requestHostDnsRefresh(host);
     }
   }
 
@@ -372,11 +371,11 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
       return false;
     }
     // Check if the Registry object's num locks has changed since this task was batched
-    int registryNumPublishLocks = Registry.get(tld).getNumDnsPublishLocks();
-    if (registryNumPublishLocks != numPublishLocks) {
+    int tldNumPublishLocks = Tld.get(tld).getNumDnsPublishLocks();
+    if (tldNumPublishLocks != numPublishLocks) {
       logger.atWarning().log(
           "Registry numDnsPublishLocks %d out of sync with parameter %d.",
-          registryNumPublishLocks, numPublishLocks);
+          tldNumPublishLocks, numPublishLocks);
       return false;
     }
     return true;