Add batching to DeleteProberDataAction (#2322)

* Add batching to DeleteProberDataAction

* Only get time once

* Add separate query for dry run

* Update querries to actually properly delete all the data

* Fix merge conflicts

* Add test for foreign key constraints

* Make transaction repeatable read

* Make queries to subtables native

* Add native query for GracePeriodHistory

* Kill job after 20 hours

* remove extra time check from read query

build.gradle

@@ -342,8 +342,8 @@ subprojects {
   // search for `flex-template-base-image` and update the parameter value.
   // There are at least two instances, one in core/build.gradle, one in
   // release/stage_beam_pipeline.sh
-  sourceCompatibility = JavaVersion.VERSION_17
-  targetCompatibility = JavaVersion.VERSION_17
+  sourceCompatibility = JavaVersion.VERSION_21
+  targetCompatibility = JavaVersion.VERSION_21
 
   project.tasks.test.dependsOn runPresubmits
 

console-webapp/package-lock.json

@@ -8639,9 +8639,9 @@
       "dev": true
     },
     "node_modules/express": {
-      "version": "4.19.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.19.1.tgz",
-      "integrity": "sha512-K4w1/Bp7y8iSiVObmCrtq8Cs79XjJc/RU2YYkZQ7wpUu5ZyZ7MtPHkqoMz4pf+mgXfNvo2qft8D9OnrH2ABk9w==",
+      "version": "4.19.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz",
+      "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==",
       "dev": true,
       "dependencies": {
         "accepts": "~1.3.8",

console-webapp/src/app/navigation/navigation.component.html

@@ -18,7 +18,7 @@
     *matTreeNodeDef="let node; when: hasChild"
     (click)="onClick(node)"
   >
-    <div class="mat-tree-node" [class.active]="router.url.endsWith(node.path)">
+    <div class="mat-tree-node" [class.active]="router.url.includes(node.path)">
       <button
         class="console-app__nav-icon_expand"
         mat-icon-button

console-webapp/src/app/registrar/registrarSelector.component.html

@@ -13,6 +13,7 @@
       [ngModel]="registrarInput()"
       (ngModelChange)="registrarInput.set($event)"
       [ngModelOptions]="{ standalone: true }"
+      (focus)="onFocus()"
       [matAutocomplete]="auto"
     />
     <mat-autocomplete

console-webapp/src/app/registrar/registrarSelector.component.ts

@@ -34,15 +34,14 @@ export class RegistrarSelectorComponent {
         option.toLowerCase().includes(filterValue)
       );
     });
-    this.onSelect(registrarService.registrarId());
+  }
+
+  onFocus() {
+    // We reset the list of options after selection, so that user doesn't have to clear it out
+    this.filteredOptions = this.allRegistrarIds();
   }
 
   onSelect(registrarId: string) {
     this.registrarService.updateSelectedRegistrar(registrarId);
-
-    // We reset the list of options after selection, so that user doesn't have to clear it out
-    setTimeout(() => {
-      this.filteredOptions = this.allRegistrarIds();
-    }, 10);
   }
 }

console-webapp/src/app/settings/contact/contact.component.scss

@@ -20,6 +20,11 @@
     mat-table {
       min-width: 800px;
     }
+    .contact__name-column-title {
+      color: #5f6368;
+      font-weight: 500;
+      padding: 10px 0;
+    }
   }
   &__empty-contacts {
     display: flex;
@@ -38,9 +43,4 @@
     right: 20px;
     top: 5px;
   }
-  .contact__name-column-title {
-    color: #5f6368;
-    font-weight: 500;
-    padding: 10px 0;
-  }
 }

console-webapp/src/app/settings/contact/contact.component.ts

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Component, effect } from '@angular/core';
+import { Component, effect, ViewEncapsulation } from '@angular/core';
 import { MatTableDataSource } from '@angular/material/table';
 import { take } from 'rxjs';
 import { RegistrarService } from 'src/app/registrar/registrar.service';
@@ -27,6 +27,7 @@ import {
   selector: 'app-contact',
   templateUrl: './contact.component.html',
   styleUrls: ['./contact.component.scss'],
+  encapsulation: ViewEncapsulation.None,
 })
 export default class ContactComponent {
   public static PATH = 'contact';
@@ -68,18 +69,18 @@ export default class ContactComponent {
     public contactService: ContactService,
     private registrarService: RegistrarService
   ) {
+    effect(() => {
+      if (this.contactService.contacts()) {
+        this.dataSource = new MatTableDataSource<ViewReadyContact>(
+          this.contactService.contacts().map(contactTypeToViewReadyContact)
+        );
+      }
+    });
     effect(() => {
       if (this.registrarService.registrarId()) {
         this.contactService.isContactDetailsView = false;
         this.contactService.isContactNewView = false;
-        this.contactService
-          .fetchContacts()
-          .pipe(take(1))
-          .subscribe((contacts) => {
-            this.dataSource = new MatTableDataSource<ViewReadyContact>(
-              contacts.map(contactTypeToViewReadyContact)
-            );
-          });
+        this.contactService.fetchContacts().pipe(take(1)).subscribe();
       }
     });
   }

console-webapp/src/app/settings/whois/whois.component.ts

@@ -36,9 +36,12 @@ export default class WhoisComponent {
     if (registrar?.localizedAddress?.state) {
       result += `${registrar?.localizedAddress?.state} `;
     }
-    if (registrar?.localizedAddress?.street) {
+    if (registrar?.localizedAddress?.countryCode) {
       result += registrar?.localizedAddress?.countryCode;
     }
+    if (registrar?.localizedAddress?.zip) {
+      result += registrar?.localizedAddress?.zip;
+    }
     return result;
   });
 

console-webapp/src/app/settings/whois/whoisEdit.component.html

@@ -51,7 +51,17 @@
       </mat-form-field>
 
       <mat-form-field appearance="outline">
-        <mat-label>Street Address: </mat-label>
+        <mat-label>Street Address (line 1): </mat-label>
+        <input
+          matInput
+          type="text"
+          [ngModelOptions]="{ standalone: true }"
+          [(ngModel)]="registrarInEdit.localizedAddress.street![0]"
+        />
+      </mat-form-field>
+
+      <mat-form-field appearance="outline">
+        <mat-label>Street Address (line 2): </mat-label>
         <input
           matInput
           type="text"

core/build.gradle

@@ -17,6 +17,7 @@ import java.util.Optional
 
 plugins {
   id 'java-library'
+  id "org.flywaydb.flyway" version "10.8.1"
   id 'maven-publish'
 }
 
@@ -317,6 +318,7 @@ dependencies {
 
   // Flyway classes needed to generate the golden file.
   implementation deps['org.flywaydb:flyway-core']
+  implementation deps['org.flywaydb:flyway-database-postgresql']
 
   closureCompiler deps['com.google.javascript:closure-compiler']
 }

core/buildscript-gradle.lockfile

@@ -1,4 +1,13 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-empty=classpath
+com.fasterxml.jackson.core:jackson-annotations:2.15.2=classpath
+com.fasterxml.jackson.core:jackson-core:2.15.2=classpath
+com.fasterxml.jackson.core:jackson-databind:2.15.2=classpath
+com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.15.2=classpath
+com.fasterxml.jackson:jackson-bom:2.15.2=classpath
+com.google.code.gson:gson:2.10.1=classpath
+gradle.plugin.org.flywaydb:gradle-plugin-publishing:10.8.1=classpath
+org.flywaydb.flyway:org.flywaydb.flyway.gradle.plugin:10.8.1=classpath
+org.flywaydb:flyway-core:10.8.1=classpath
+empty=

core/gradle.lockfile

@@ -407,7 +407,7 @@ org.eclipse.jetty:jetty-session:12.0.7=testCompileClasspath,testRuntimeClasspath
 org.eclipse.jetty:jetty-util:12.0.7=testCompileClasspath,testRuntimeClasspath
 org.eclipse.jetty:jetty-xml:12.0.7=testCompileClasspath,testRuntimeClasspath
 org.flywaydb:flyway-core:10.10.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.flywaydb:flyway-database-postgresql:10.10.0=testRuntimeClasspath
+org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:txw2:2.3.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.gwtproject:gwt-user:2.10.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -482,7 +482,7 @@ org.ow2.asm:asm:7.0=soy
 org.ow2.asm:asm:9.2=compileClasspath,nonprodCompileClasspath
 org.ow2.asm:asm:9.6=deploy_jar,jacocoAnt,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.pcollections:pcollections:3.1.4=annotationProcessor,errorprone,nonprodAnnotationProcessor,testAnnotationProcessor
-org.postgresql:postgresql:42.7.3=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.7.2=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.reflections:reflections:0.10.2=checkstyle
 org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.seleniumhq.selenium:selenium-api:3.141.59=testCompileClasspath,testRuntimeClasspath

core/src/main/java/google/registry/batch/DeleteProberDataAction.java

@@ -16,15 +16,19 @@ package google.registry.batch;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.dns.DnsUtils.requestDomainDnsRefresh;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
 import static google.registry.model.tld.Tlds.getTldsOfType;
+import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
+import static google.registry.request.RequestParameters.PARAM_BATCH_SIZE;
 import static google.registry.request.RequestParameters.PARAM_DRY_RUN;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
 import static google.registry.util.RegistryEnvironment.PRODUCTION;
+import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
@@ -41,12 +45,11 @@ import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
 import google.registry.util.RegistryEnvironment;
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import javax.inject.Inject;
-import org.hibernate.CacheMode;
-import org.hibernate.ScrollMode;
-import org.hibernate.ScrollableResults;
-import org.hibernate.query.Query;
+import javax.persistence.TypedQuery;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
@@ -61,6 +64,7 @@ import org.joda.time.Duration;
     auth = Auth.AUTH_API_ADMIN)
 public class DeleteProberDataAction implements Runnable {
 
+  // TODO(b/323026070): Add email alert on failure of this action
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   /**
@@ -90,31 +94,38 @@ public class DeleteProberDataAction implements Runnable {
   // Note: creationTime must be compared to a Java object (CreateAutoTimestamp) but deletionTime can
   // be compared directly to the SQL timestamp (it's a DateTime)
   private static final String DOMAIN_QUERY_STRING =
-      "FROM Domain d WHERE d.tld IN :tlds AND d.domainName NOT LIKE 'nic.%' AND"
+      "FROM Domain d WHERE d.tld IN :tlds AND d.domainName NOT LIKE 'nic.%%' AND"
           + " (d.subordinateHosts IS EMPTY OR d.subordinateHosts IS NULL) AND d.creationTime <"
-          + " :creationTimeCutoff AND ((d.creationTime <= :nowAutoTimestamp AND d.deletionTime >"
-          + " current_timestamp()) OR d.deletionTime < :nowMinusSoftDeleteDelay) ORDER BY d.repoId";
+          + " :creationTimeCutoff AND (d.deletionTime > :now OR d.deletionTime <"
+          + " :nowMinusSoftDeleteDelay)";
 
   /** Number of domains to retrieve and delete per SQL transaction. */
-  private static final int BATCH_SIZE = 1000;
+  private static final int DEFAULT_BATCH_SIZE = 1000;
 
-  @Inject
-  @Parameter(PARAM_DRY_RUN)
   boolean isDryRun;
+
   /** List of TLDs to work on. If empty - will work on all TLDs that end with .test. */
-  @Inject
-  @Parameter(PARAM_TLDS)
   ImmutableSet<String> tlds;
 
-  @Inject
-  @Config("registryAdminClientId")
+  int batchSize;
+
   String registryAdminRegistrarId;
 
   @Inject
-  DeleteProberDataAction() {}
+  DeleteProberDataAction(
+      @Parameter(PARAM_DRY_RUN) boolean isDryRun,
+      @Parameter(PARAM_TLDS) ImmutableSet<String> tlds,
+      @Parameter(PARAM_BATCH_SIZE) Optional<Integer> batchSize,
+      @Config("registryAdminClientId") String registryAdminRegistrarId) {
+    this.isDryRun = isDryRun;
+    this.tlds = tlds;
+    this.batchSize = batchSize.orElse(DEFAULT_BATCH_SIZE);
+    this.registryAdminRegistrarId = registryAdminRegistrarId;
+  }
 
   @Override
   public void run() {
+    checkArgument(batchSize > 0, "The batch size must be greater than 0");
     checkState(
         !Strings.isNullOrEmpty(registryAdminRegistrarId),
         "Registry admin client ID must be configured for prober data deletion to work");
@@ -131,13 +142,30 @@ public class DeleteProberDataAction implements Runnable {
         "If tlds are given, they must all exist and be TEST tlds. Given: %s, not found: %s",
         tlds,
         Sets.difference(tlds, deletableTlds));
-    runSqlJob(deletableTlds);
-  }
-
-  private void runSqlJob(ImmutableSet<String> deletableTlds) {
     AtomicInteger softDeletedDomains = new AtomicInteger();
     AtomicInteger hardDeletedDomains = new AtomicInteger();
-    tm().transact(() -> processDomains(deletableTlds, softDeletedDomains, hardDeletedDomains));
+    AtomicReference<ImmutableList<Domain>> domainsBatch = new AtomicReference<>();
+    DateTime startTime = DateTime.now(UTC);
+    do {
+      tm().transact(
+              TRANSACTION_REPEATABLE_READ,
+              () ->
+                  domainsBatch.set(
+                      processDomains(
+                          deletableTlds, softDeletedDomains, hardDeletedDomains, startTime)));
+
+      // Only process one batch in dryrun mode
+      if (isDryRun) {
+        break;
+      }
+
+      logger.atInfo().log(
+          "deleteProberData hard deleted %d names with %d batchSize",
+          hardDeletedDomains.get(), batchSize);
+
+      // Automatically kill the job if it is running for over 20 hours
+    } while (DateTime.now(UTC).isBefore(startTime.plusHours(20))
+        && domainsBatch.get().size() == batchSize);
     logger.atInfo().log(
         "%s %d domains.",
         isDryRun ? "Would have soft-deleted" : "Soft-deleted", softDeletedDomains.get());
@@ -146,46 +174,32 @@ public class DeleteProberDataAction implements Runnable {
         isDryRun ? "Would have hard-deleted" : "Hard-deleted", hardDeletedDomains.get());
   }
 
-  private void processDomains(
+  private ImmutableList<Domain> processDomains(
       ImmutableSet<String> deletableTlds,
       AtomicInteger softDeletedDomains,
-      AtomicInteger hardDeletedDomains) {
-    DateTime now = tm().getTransactionTime();
-    // Scroll through domains, soft-deleting as necessary (very few will be soft-deleted) and
-    // keeping track of which domains to hard-delete (there can be many, so we batch them up)
-    try (ScrollableResults scrollableResult =
+      AtomicInteger hardDeletedDomains,
+      DateTime now) {
+    TypedQuery<Domain> query =
         tm().query(DOMAIN_QUERY_STRING, Domain.class)
             .setParameter("tlds", deletableTlds)
             .setParameter(
                 "creationTimeCutoff", CreateAutoTimestamp.create(now.minus(DOMAIN_USED_DURATION)))
             .setParameter("nowMinusSoftDeleteDelay", now.minus(SOFT_DELETE_DELAY))
-            .setParameter("nowAutoTimestamp", CreateAutoTimestamp.create(now))
-            .unwrap(Query.class)
-            .setCacheMode(CacheMode.IGNORE)
-            .scroll(ScrollMode.FORWARD_ONLY)) {
+            .setParameter("now", now);
+    ImmutableList<Domain> domainList =
+        query.setMaxResults(batchSize).getResultStream().collect(toImmutableList());
       ImmutableList.Builder<String> domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
       ImmutableList.Builder<String> hostNamesToHardDelete = new ImmutableList.Builder<>();
-      for (int i = 1; scrollableResult.next(); i = (i + 1) % BATCH_SIZE) {
-        Domain domain = (Domain) scrollableResult.get(0);
-        processDomain(
-            domain,
-            domainRepoIdsToHardDelete,
-            hostNamesToHardDelete,
-            softDeletedDomains,
-            hardDeletedDomains);
-        // Batch the deletion and DB flush + session clearing, so we don't OOM
-        if (i == 0) {
-          hardDeleteDomainsAndHosts(
-              domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
-          domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
-          hostNamesToHardDelete = new ImmutableList.Builder<>();
-          tm().getEntityManager().flush();
-          tm().getEntityManager().clear();
-        }
-      }
-      // process the remainder
-      hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+    for (Domain domain : domainList) {
+      processDomain(
+          domain,
+          domainRepoIdsToHardDelete,
+          hostNamesToHardDelete,
+          softDeletedDomains,
+          hardDeletedDomains);
     }
+    hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+    return domainList;
   }
 
   private void processDomain(
@@ -225,6 +239,10 @@ public class DeleteProberDataAction implements Runnable {
     tm().query("DELETE FROM Host WHERE hostName IN :hostNames")
         .setParameter("hostNames", hostNames)
         .executeUpdate();
+    tm().getEntityManager()
+        .createNativeQuery("DELETE FROM \"GracePeriod\" WHERE domain_repo_id IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
     tm().query("DELETE FROM BillingEvent WHERE domainRepoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
@@ -234,12 +252,45 @@ public class DeleteProberDataAction implements Runnable {
     tm().query("DELETE FROM BillingCancellation WHERE domainRepoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
-    tm().query("DELETE FROM DomainHistory WHERE repoId IN :repoIds")
+    tm().getEntityManager()
+        .createNativeQuery("DELETE FROM \"GracePeriodHistory\" WHERE domain_repo_id IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
     tm().query("DELETE FROM PollMessage WHERE domainRepoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();
+    tm().getEntityManager()
+        .createNativeQuery("DELETE FROM \"DomainHost\" WHERE domain_repo_id IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    tm().getEntityManager()
+        .createNativeQuery(
+            "DELETE FROM \"DomainHistoryHost\" WHERE domain_history_domain_repo_id IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    tm().getEntityManager()
+        .createNativeQuery("DELETE FROM \"HostHistory\" WHERE host_name IN :hostNames")
+        .setParameter("hostNames", hostNames)
+        .executeUpdate();
+    tm().getEntityManager()
+        .createNativeQuery("DELETE FROM \"DelegationSignerData\" WHERE domain_repo_id IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    tm().getEntityManager()
+        .createNativeQuery(
+            "DELETE FROM \"DomainTransactionRecord\" WHERE domain_repo_id IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    tm().getEntityManager()
+        .createNativeQuery("DELETE FROM \"DomainDsDataHistory\" WHERE domain_repo_id IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    tm().getEntityManager()
+        .createNativeQuery("DELETE FROM \"DomainHistory\" WHERE domain_repo_id IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    // Delete from domain table is done last so that a trainsient failure in the middle of an action
+    // run will not prevent the domain from being deleted by this action in a future run
     tm().query("DELETE FROM Domain WHERE repoId IN :repoIds")
         .setParameter("repoIds", domainRepoIds)
         .executeUpdate();

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -91,11 +91,13 @@ public class DomainHistory extends HistoryEntry {
   @ElementCollection
   @JoinTable(
       name = "DomainHistoryHost",
-      indexes =
-          @Index(
-              columnList =
-                  "domain_history_history_revision_id,domain_history_domain_repo_id,host_repo_id",
-              unique = true))
+      indexes = {
+        @Index(columnList = "domain_history_domain_repo_id"),
+        @Index(
+            columnList =
+                "domain_history_history_revision_id,domain_history_domain_repo_id,host_repo_id",
+            unique = true)
+      })
   @Column(name = "host_repo_id")
   Set<VKey<Host>> nsHosts;
 

core/src/main/java/google/registry/model/domain/secdns/DomainDsData.java

@@ -36,7 +36,7 @@ import javax.xml.bind.annotation.XmlType;
  * @see <a href="http://tools.ietf.org/html/rfc4034">RFC 4034</a>
  */
 @XmlType(name = "dsData")
-@Entity
+@Entity(name = "DelegationSignerData")
 @IdClass(DomainDsDataId.class)
 @Table(name = "DelegationSignerData", indexes = @Index(columnList = "domainRepoId"))
 public class DomainDsData extends DomainDsDataBase {

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -86,7 +86,12 @@ import org.joda.time.DateTime;
 @ExternalMessagingName("message")
 @Inheritance(strategy = InheritanceType.SINGLE_TABLE)
 @DiscriminatorColumn(name = "type")
-@Table(indexes = {@Index(columnList = "registrar_id"), @Index(columnList = "eventTime")})
+@Table(
+    indexes = {
+      @Index(columnList = "domainRepoId"),
+      @Index(columnList = "registrar_id"),
+      @Index(columnList = "eventTime")
+    })
 public abstract class PollMessage extends ImmutableObject
     implements Buildable, TransferServerApproveEntity, UnsafeSerializable {
 

core/src/main/java/google/registry/request/RequestModule.java

@@ -18,8 +18,10 @@ import static com.google.common.net.MediaType.JSON_UTF_8;
 import static google.registry.dns.PublishDnsUpdatesAction.CLOUD_TASKS_RETRY_HEADER;
 import static google.registry.model.tld.Tlds.assertTldExists;
 import static google.registry.model.tld.Tlds.assertTldsExist;
+import static google.registry.request.RequestParameters.PARAM_BATCH_SIZE;
 import static google.registry.request.RequestParameters.PARAM_DRY_RUN;
 import static google.registry.request.RequestParameters.extractBooleanParameter;
+import static google.registry.request.RequestParameters.extractOptionalIntParameter;
 import static google.registry.request.RequestParameters.extractRequiredHeader;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 import static google.registry.request.RequestParameters.extractSetOfParameters;
@@ -99,6 +101,12 @@ public final class RequestModule {
     return extractBooleanParameter(req, PARAM_DRY_RUN);
   }
 
+  @Provides
+  @Parameter(PARAM_BATCH_SIZE)
+  static Optional<Integer> provideBatchSize(HttpServletRequest req) {
+    return extractOptionalIntParameter(req, PARAM_BATCH_SIZE);
+  }
+
   @Provides
   static Response provideResponse(ResponseImpl response) {
     return response;

core/src/main/java/google/registry/request/RequestParameters.java

@@ -41,6 +41,11 @@ public final class RequestParameters {
   /** The standardized request parameter name used by any action in dry run mode. */
   public static final String PARAM_DRY_RUN = "dryRun";
 
+  /**
+   * The standardized request parameter name used by any action taking a configurable batch size.
+   */
+  public static final String PARAM_BATCH_SIZE = "batchSize";
+
   /**
    * Returns first GET or POST parameter associated with {@code name}.
    *

core/src/main/java/google/registry/tools/server/RefreshDnsForAllDomainsAction.java

@@ -21,6 +21,7 @@ import static google.registry.dns.DnsUtils.requestDomainDnsRefresh;
 import static google.registry.model.tld.Tlds.assertTldsExist;
 import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.request.RequestParameters.PARAM_BATCH_SIZE;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
@@ -83,7 +84,7 @@ public class RefreshDnsForAllDomainsAction implements Runnable {
   RefreshDnsForAllDomainsAction(
       Response response,
       @Parameter(PARAM_TLDS) ImmutableSet<String> tlds,
-      @Parameter("batchSize") Optional<Integer> batchSize,
+      @Parameter(PARAM_BATCH_SIZE) Optional<Integer> batchSize,
       @Parameter("refreshQps") Optional<Integer> refreshQps,
       Random random) {
     this.response = response;

core/src/main/java/google/registry/tools/server/ToolsServerModule.java

@@ -70,12 +70,6 @@ public class ToolsServerModule {
     return extractRequiredParameter(req, "rawKeys");
   }
 
-  @Provides
-  @Parameter("batchSize")
-  static Optional<Integer> provideBatchSize(HttpServletRequest req) {
-    return extractOptionalIntParameter(req, "batchSize");
-  }
-
   @Provides
   @Parameter("refreshQps")
   static Optional<Integer> provideRefreshQps(HttpServletRequest req) {

core/src/nonprod/java/google/registry/tools/DumpGoldenSchemaCommand.java

@@ -19,6 +19,7 @@ import com.beust.jcommander.Parameters;
 import java.io.IOException;
 import java.nio.file.Path;
 import org.flywaydb.core.Flyway;
+import org.flywaydb.database.postgresql.PostgreSQLConfigurationExtension;
 import org.testcontainers.containers.Container;
 
 /**
@@ -50,6 +51,10 @@ public class DumpGoldenSchemaCommand extends PostgresqlCommand {
                 postgresContainer.getUsername(),
                 postgresContainer.getPassword())
             .load();
+
+    PostgreSQLConfigurationExtension configurationExtension =
+        flyway.getConfigurationExtension(PostgreSQLConfigurationExtension.class);
+    configurationExtension.setTransactionalLock(false);
     flyway.migrate();
 
     String userName = postgresContainer.getUsername();

core/src/test/java/google/registry/batch/DeleteProberDataActionTest.java

@@ -17,6 +17,7 @@ package google.registry.batch;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
+import static google.registry.model.domain.rgp.GracePeriodStatus.ADD;
 import static google.registry.testing.DatabaseHelper.assertDomainDnsRequests;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.loadByEntitiesIfPresent;
@@ -37,6 +38,7 @@ import google.registry.model.billing.BillingBase.Reason;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.GracePeriod;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Tld;
@@ -92,11 +94,7 @@ class DeleteProberDataActionTest {
   }
 
   private void resetAction() {
-    action = new DeleteProberDataAction();
-    action.isDryRun = false;
-    action.tlds = ImmutableSet.of();
-    action.registryAdminRegistrarId = "TheRegistrar";
-    // RegistryEnvironment.SANDBOX.setup(systemPropertyExtension);
+    action = new DeleteProberDataAction(false, ImmutableSet.of(), Optional.empty(), "TheRegistrar");
   }
 
   @AfterEach
@@ -119,6 +117,19 @@ class DeleteProberDataActionTest {
     assertAllAbsent(oaEntities);
   }
 
+  @Test
+  void test_deletesAllInBatches() throws Exception {
+    // Persist 40 domains
+    Set<ImmutableObject> ibEntities = persistLotsOfDomains("ib-any.test");
+    Set<ImmutableObject> oaEntities = persistLotsOfDomains("oa-canary.test");
+    // Create action with batch size of 3
+    DeleteProberDataAction batchedAction =
+        new DeleteProberDataAction(false, ImmutableSet.of(), Optional.of(3), "TheRegistrar");
+    batchedAction.run();
+    assertAllAbsent(ibEntities);
+    assertAllAbsent(oaEntities);
+  }
+
   @Test
   void testSuccess_deletesAllAndOnlyGivenTlds() throws Exception {
     Set<ImmutableObject> tldEntities = persistLotsOfDomains("tld");
@@ -303,12 +314,22 @@ class DeleteProberDataActionTest {
                 .setRegistrarId("TheRegistrar")
                 .setMsg("Domain registered")
                 .build());
+    GracePeriod gracePeriod =
+        persistSimpleResource(
+            GracePeriod.create(
+                ADD,
+                domain.getRepoId(),
+                DELETION_TIME.plusDays(5),
+                "TheRegistrar",
+                billingEvent.createVKey()));
+    domain = persistResource(domain.asBuilder().addGracePeriod(gracePeriod).build());
     ImmutableSet.Builder<ImmutableObject> builder =
         new ImmutableSet.Builder<ImmutableObject>()
             .add(domain)
             .add(historyEntry)
             .add(billingEvent)
-            .add(pollMessage);
+            .add(pollMessage)
+            .add(gracePeriod);
     return builder.build();
   }
 

db/build.gradle

@@ -147,6 +147,9 @@ publishing {
 
 flyway {
   def accessInfo = project.ext.getJdbcAccessInfo()
+  pluginConfiguration = [
+          postgresqlTransactionalLock: 'false'
+  ]
 
   url = accessInfo.url
   user = accessInfo.user
@@ -160,7 +163,7 @@ dependencies {
   def deps = rootProject.dependencyMap
 
   implementation deps['org.flywaydb:flyway-core']
-  runtimeOnly deps['org.flywaydb:flyway-database-postgresql']
+  implementation deps['org.flywaydb:flyway-database-postgresql']
 
   runtimeOnly deps['com.google.cloud.sql:postgres-socket-factory']
   runtimeOnly deps['org.postgresql:postgresql']

db/gradle.lockfile

@@ -91,7 +91,7 @@ org.checkerframework:checker-qual:3.12.0=checkstyle
 org.checkerframework:checker-qual:3.33.0=annotationProcessor,errorprone,testAnnotationProcessor
 org.checkerframework:checker-qual:3.42.0=deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.flywaydb:flyway-core:10.10.0=compileClasspath,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.flywaydb:flyway-database-postgresql:10.10.0=deploy_jar,runtimeClasspath,testRuntimeClasspath
+org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.hamcrest:hamcrest-core:1.3=testCompileClasspath,testRuntimeClasspath
 org.jacoco:org.jacoco.agent:0.8.11=jacocoAgent,jacocoAnt
 org.jacoco:org.jacoco.ant:0.8.11=jacocoAnt
@@ -113,7 +113,7 @@ org.ow2.asm:asm-tree:9.6=jacocoAnt
 org.ow2.asm:asm:9.2=deploy_jar,runtimeClasspath
 org.ow2.asm:asm:9.6=jacocoAnt,testCompileClasspath,testRuntimeClasspath
 org.pcollections:pcollections:3.1.4=annotationProcessor,errorprone,testAnnotationProcessor
-org.postgresql:postgresql:42.7.3=deploy_jar,runtimeClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.7.2=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.reflections:reflections:0.10.2=checkstyle
 org.rnorth.duct-tape:duct-tape:1.0.8=testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:1.7.36=testCompileClasspath

db/src/main/resources/sql/er_diagram/brief_er_diagram.html

@@ -261,11 +261,11 @@ td.section {
     </tr>
     <tr>
      <td class="property_name">generated on</td>
-     <td class="property_value">2024-03-11 20:42:54.463293864</td>
+     <td class="property_value">2024-03-28 21:10:10.784013213</td>
     </tr>
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V164__rename_console_user_history_time.sql</td>
+     <td id="lastFlywayFile" class="property_value">V166__drop_should_publish_column.sql</td>
     </tr>
    </tbody>
   </table>
@@ -280,7 +280,7 @@ td.section {
     <text text-anchor="start" x="3730.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated by</text>
     <text text-anchor="start" x="3813.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">SchemaCrawler 16.10.1</text>
     <text text-anchor="start" x="3729.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated on</text>
-    <text text-anchor="start" x="3813.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">2024-03-11 20:42:54.463293864</text>
+    <text text-anchor="start" x="3813.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">2024-03-28 21:10:10.784013213</text>
     <polygon fill="none" stroke="#888888" points="3726,-4 3726,-44 4013,-44 4013,-4 3726,-4" /> <!-- allocationtoken_a08ccbef -->
     <g id="node1" class="node">
      <title>

db/src/main/resources/sql/er_diagram/full_er_diagram.html

@@ -261,11 +261,11 @@ td.section {
     </tr>
     <tr>
      <td class="property_name">generated on</td>
-     <td class="property_value">2024-03-11 20:42:51.738018408</td>
+     <td class="property_value">2024-03-28 21:10:08.34519385</td>
     </tr>
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V164__rename_console_user_history_time.sql</td>
+     <td id="lastFlywayFile" class="property_value">V166__drop_should_publish_column.sql</td>
     </tr>
    </tbody>
   </table>
@@ -277,11 +277,11 @@ td.section {
      SchemaCrawler_Diagram
     </title>
     <polygon fill="white" stroke="transparent" points="-4,4 -4,-9310 4738,-9310 4738,4 -4,4" />
-    <text text-anchor="start" x="4443.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated by</text>
-    <text text-anchor="start" x="4526.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">SchemaCrawler 16.10.1</text>
-    <text text-anchor="start" x="4442.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated on</text>
-    <text text-anchor="start" x="4526.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">2024-03-11 20:42:51.738018408</text>
-    <polygon fill="none" stroke="#888888" points="4439,-4 4439,-44 4726,-44 4726,-4 4439,-4" /> <!-- allocationtoken_a08ccbef -->
+    <text text-anchor="start" x="4450" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated by</text>
+    <text text-anchor="start" x="4533" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">SchemaCrawler 16.10.1</text>
+    <text text-anchor="start" x="4449" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated on</text>
+    <text text-anchor="start" x="4533" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">2024-03-28 21:10:08.34519385</text>
+    <polygon fill="none" stroke="#888888" points="4446,-4 4446,-44 4726,-44 4726,-4 4446,-4" /> <!-- allocationtoken_a08ccbef -->
     <g id="node1" class="node">
      <title>
       allocationtoken_a08ccbef
@@ -3882,10 +3882,7 @@ td.section {
      <text text-anchor="start" x="3684.5" y="-8699.3" font-family="Helvetica,sans-Serif" font-size="14.00">name</text>
      <text text-anchor="start" x="3808.5" y="-8699.3" font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
      <text text-anchor="start" x="3817.5" y="-8699.3" font-family="Helvetica,sans-Serif" font-size="14.00">text not null</text>
-     <text text-anchor="start" x="3684.5" y="-8680.3" font-family="Helvetica,sans-Serif" font-size="14.00">should_publish</text>
-     <text text-anchor="start" x="3808.5" y="-8680.3" font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
-     <text text-anchor="start" x="3817.5" y="-8680.3" font-family="Helvetica,sans-Serif" font-size="14.00">bool</text>
-     <polygon fill="none" stroke="#888888" points="3681,-8673.5 3681,-8789.5 3942,-8789.5 3942,-8673.5 3681,-8673.5" />
+     <polygon fill="none" stroke="#888888" points="3681,-8693 3681,-8790 3942,-8790 3942,-8693 3681,-8693" />
     </g> <!-- reservedentry_1a7b8520&#45;&gt;reservedlist_b97c3f1c -->
     <g id="edge81" class="edge">
      <title>
@@ -8671,6 +8668,18 @@ td.section {
     <tr>
      <td colspan="3"></td>
     </tr>
+    <tr>
+     <td colspan="2" class="name">idx69qun5kxt3eux5igrxrqcycv0</td>
+     <td class="description right">[non-unique index]</td>
+    </tr>
+    <tr>
+     <td class="spacer"></td>
+     <td class="minwidth">domain_history_domain_repo_id</td>
+     <td class="minwidth">ascending</td>
+    </tr>
+    <tr>
+     <td colspan="3"></td>
+    </tr>
     <tr>
      <td colspan="2" class="name">ukt2e7ae3t8gcsxd13wjx2ka7ij</td>
      <td class="description right">[unique index]</td>
@@ -10253,6 +10262,18 @@ td.section {
     <tr>
      <td colspan="3"></td>
     </tr>
+    <tr>
+     <td colspan="2" class="name">idxf2q9dqj899h1q8lah5y719nxd</td>
+     <td class="description right">[non-unique index]</td>
+    </tr>
+    <tr>
+     <td class="spacer"></td>
+     <td class="minwidth">domain_repo_id</td>
+     <td class="minwidth">ascending</td>
+    </tr>
+    <tr>
+     <td colspan="3"></td>
+    </tr>
     <tr>
      <td colspan="2" class="name">idxe7wu46c7wpvfmfnj4565abibp</td>
      <td class="description right">[non-unique index]</td>
@@ -12210,11 +12231,6 @@ td.section {
      <td class="minwidth">name</td>
      <td class="minwidth">text not null</td>
     </tr>
-    <tr>
-     <td class="spacer"></td>
-     <td class="minwidth">should_publish</td>
-     <td class="minwidth">bool</td>
-    </tr>
     <tr>
      <td colspan="3"></td>
     </tr>

db/src/main/resources/sql/flyway.txt

@@ -162,3 +162,5 @@ V161__rename_console_epp_history_time.sql
 V162__rename_console_poc_history_time.sql
 V163__rename_console_registrar_history_time.sql
 V164__rename_console_user_history_time.sql
+V165__add_domain_repo_id_indexes_to_more_tables.sql
+V166__drop_should_publish_column.sql

db/src/main/resources/sql/flyway/V165__add_domain_repo_id_indexes_to_more_tables.sql

@@ -0,0 +1,16 @@
+-- Copyright 2024 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX CONCURRENTLY IF NOT EXISTS IDX69qun5kxt3eux5igrxrqcycv0 ON "DomainHistoryHost" (domain_history_domain_repo_id);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS IDXf2q9dqj899h1q8lah5y719nxd ON "PollMessage" (domain_repo_id);

db/src/main/resources/sql/flyway/V166__drop_should_publish_column.sql

@@ -0,0 +1,15 @@
+-- Copyright 2024 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "ReservedList" drop column should_publish;

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -848,6 +848,7 @@ create index IDXrh4xmrot9bd63o382ow9ltfig on "DomainHistory" (creation_time);
 create index IDXaro1omfuaxjwmotk3vo00trwm on "DomainHistory" (history_registrar_id);
 create index IDXsu1nam10cjes9keobapn5jvxj on "DomainHistory" (history_type);
 create index IDX6w3qbtgce93cal2orjg1tw7b7 on "DomainHistory" (history_modification_time);
+create index IDX69qun5kxt3eux5igrxrqcycv0 on "DomainHistoryHost" (domain_history_domain_repo_id);
 
     alter table if exists "DomainHistoryHost" 
        add constraint UKt2e7ae3t8gcsxd13wjx2ka7ij unique (domain_history_history_revision_id, domain_history_domain_repo_id, host_repo_id);
@@ -869,6 +870,7 @@ create index IDXkkwbwcwvrdkkqothkiye4jiff on "HostHistory" (host_name);
 create index IDXknk8gmj7s47q56cwpa6rmpt5l on "HostHistory" (history_type);
 create index IDX67qwkjtlq5q8dv6egtrtnhqi7 on "HostHistory" (history_modification_time);
 create index IDXlg6a5tp70nch9cp0gc11brc5o on "PackagePromotion" (token);
+create index IDXf2q9dqj899h1q8lah5y719nxd on "PollMessage" (domain_repo_id);
 create index IDXe7wu46c7wpvfmfnj4565abibp on "PollMessage" (registrar_id);
 create index IDXaydgox62uno9qx8cjlj5lauye on "PollMessage" (event_time);
 create index premiumlist_name_idx on "PremiumList" (name);

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -1129,8 +1129,7 @@ CREATE TABLE public."ReservedEntry" (
 CREATE TABLE public."ReservedList" (
     revision_id bigint NOT NULL,
     creation_timestamp timestamp with time zone NOT NULL,
-    name text NOT NULL,
-    should_publish boolean
+    name text NOT NULL
 );
 
 
@@ -1961,6 +1960,13 @@ CREATE INDEX idx5yqacw829y5bm6f7eajsq1cts ON public."UserUpdateHistory" USING bt
 CREATE INDEX idx67qwkjtlq5q8dv6egtrtnhqi7 ON public."HostHistory" USING btree (history_modification_time);
 
 
+--
+-- Name: idx69qun5kxt3eux5igrxrqcycv0; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx69qun5kxt3eux5igrxrqcycv0 ON public."DomainHistoryHost" USING btree (domain_history_domain_repo_id);
+
+
 --
 -- Name: idx6ebt3nwk5ocvnremnhnlkl6ff; Type: INDEX; Schema: public; Owner: -
 --
@@ -2129,6 +2135,13 @@ CREATE INDEX idxe7wu46c7wpvfmfnj4565abibp ON public."PollMessage" USING btree (r
 CREATE INDEX idxeokttmxtpq2hohcioe5t2242b ON public."BillingCancellation" USING btree (registrar_id);
 
 
+--
+-- Name: idxf2q9dqj899h1q8lah5y719nxd; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxf2q9dqj899h1q8lah5y719nxd ON public."PollMessage" USING btree (domain_repo_id);
+
+
 --
 -- Name: idxfdk2xpil2x1gh0omt84k2y3o1; Type: INDEX; Schema: public; Owner: -
 --

db/src/test/java/google/registry/sql/flyway/SchemaTest.java

@@ -26,6 +26,7 @@ import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
 import org.flywaydb.core.Flyway;
+import org.flywaydb.database.postgresql.PostgreSQLConfigurationExtension;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
 import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
@@ -92,6 +93,10 @@ class SchemaTest {
                 sqlContainer.getJdbcUrl(), sqlContainer.getUsername(), sqlContainer.getPassword())
             .load();
 
+    PostgreSQLConfigurationExtension configurationExtension =
+        flyway.getConfigurationExtension(PostgreSQLConfigurationExtension.class);
+    configurationExtension.setTransactionalLock(false);
+
     // flyway.migrate() returns the number of newly pushed scripts. This is a variable
     // number as our schema evolves.
     assertThat(flyway.migrate().migrations).isNotEmpty();
@@ -123,6 +128,10 @@ class SchemaTest {
             .dataSource(
                 sqlContainer.getJdbcUrl(), sqlContainer.getUsername(), sqlContainer.getPassword())
             .load();
+    PostgreSQLConfigurationExtension configurationExtension =
+        flyway.getConfigurationExtension(PostgreSQLConfigurationExtension.class);
+    configurationExtension.setTransactionalLock(false);
+
     flyway.migrate();
     logger.atInfo().log("Base schema version: %s", flyway.info().current().getVersion());
 
@@ -133,6 +142,9 @@ class SchemaTest {
             .dataSource(
                 sqlContainer.getJdbcUrl(), sqlContainer.getUsername(), sqlContainer.getPassword())
             .load();
+    configurationExtension =
+        flyway.getConfigurationExtension(PostgreSQLConfigurationExtension.class);
+    configurationExtension.setTransactionalLock(false);
     flyway.migrate();
     flyway.validate();
     logger.atInfo().log("Latest schema version: %s", flyway.info().current().getVersion());

gradle.properties

@@ -1,4 +1,4 @@
-# This file defines properties used by the gradle build.  It must be kept in
+# This file defines properties used by the gradle build. It must be kept in
 # sync with config/nom_build.py.
 #
 # To regenerate, run ./nom_build --generate-gradle-properties

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew.bat

@@ -43,11 +43,11 @@ set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
 if %ERRORLEVEL% equ 0 goto execute
 
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 
@@ -57,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
 if exist "%JAVA_EXE%" goto execute
 
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 

jetty/gradle.lockfile

@@ -324,6 +324,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=deploy_jar,runtimeClasspath,testR
 org.codehaus.mojo:animal-sniffer-annotations:1.23=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.conscrypt:conscrypt-openjdk-uber:2.5.2=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.flywaydb:flyway-core:10.10.0=deploy_jar,runtimeClasspath,testRuntimeClasspath
+org.flywaydb:flyway-database-postgresql:10.10.0=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-runtime:2.3.1=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.glassfish.jaxb:txw2:2.3.1=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.gwtproject:gwt-user:2.10.0=deploy_jar,runtimeClasspath,testRuntimeClasspath
@@ -366,7 +367,7 @@ org.ow2.asm:asm-tree:9.6=jacocoAnt
 org.ow2.asm:asm-util:7.0=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.6=deploy_jar,jacocoAnt,runtimeClasspath,testRuntimeClasspath
 org.pcollections:pcollections:3.1.4=annotationProcessor,errorprone,testAnnotationProcessor
-org.postgresql:postgresql:42.7.3=deploy_jar,runtimeClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.7.2=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.reflections:reflections:0.10.2=checkstyle
 org.rnorth.duct-tape:duct-tape:1.0.8=deploy_jar,runtimeClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:2.1.0-alpha1=deploy_jar,runtimeClasspath,testRuntimeClasspath

processor/src/main/java/google/registry/processors/VKeyProcessor.java

@@ -50,7 +50,7 @@ import javax.persistence.Converter;
 
 /** Processor to generate {@link AttributeConverter} for {@code VKey} type. */
 @SupportedAnnotationTypes("google.registry.persistence.WithVKey")
-@SupportedSourceVersion(SourceVersion.RELEASE_17)
+@SupportedSourceVersion(SourceVersion.RELEASE_21)
 public class VKeyProcessor extends AbstractProcessor {
 
   private static final String CONVERTER_CLASS_NAME_TEMP = "VKeyConverter_%s";

release/schema-deployer/deploy_sql_schema.sh

@@ -78,7 +78,8 @@ if ! pgrep cloud_sql_proxy; then
   exit 1
 fi
 
-/flyway/flyway -community -user=${db_user} -password=${db_password} \
+/flyway/flyway -postgresql.transactional.lock=false -community \
+  -user=${db_user} -password=${db_password} \
   -url=jdbc:postgresql://localhost:5432/postgres \
   -locations=classpath:sql/flyway \
   ${flyway_action}

services/backend/gradle.lockfile

@@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath
 org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath
 org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -338,7 +339,7 @@ org.ow2.asm:asm-tree:9.2=compileClasspath,runtimeClasspath,testCompileClasspath,
 org.ow2.asm:asm-util:7.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.2=compileClasspath,testCompileClasspath
 org.ow2.asm:asm:9.6=runtimeClasspath,testRuntimeClasspath
-org.postgresql:postgresql:42.7.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.7.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-jdk14:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

services/bsa/gradle.lockfile

@@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath
 org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath
 org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -338,7 +339,7 @@ org.ow2.asm:asm-tree:9.2=compileClasspath,runtimeClasspath,testCompileClasspath,
 org.ow2.asm:asm-util:7.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.2=compileClasspath,testCompileClasspath
 org.ow2.asm:asm:9.6=runtimeClasspath,testRuntimeClasspath
-org.postgresql:postgresql:42.7.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.7.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-jdk14:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

services/default/gradle.lockfile

@@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath
 org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath
 org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -338,7 +339,7 @@ org.ow2.asm:asm-tree:9.2=compileClasspath,runtimeClasspath,testCompileClasspath,
 org.ow2.asm:asm-util:7.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.2=compileClasspath,testCompileClasspath
 org.ow2.asm:asm:9.6=runtimeClasspath,testRuntimeClasspath
-org.postgresql:postgresql:42.7.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.7.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-jdk14:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

services/pubapi/gradle.lockfile

@@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath
 org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath
 org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -338,7 +339,7 @@ org.ow2.asm:asm-tree:9.2=compileClasspath,runtimeClasspath,testCompileClasspath,
 org.ow2.asm:asm-util:7.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.2=compileClasspath,testCompileClasspath
 org.ow2.asm:asm:9.6=runtimeClasspath,testRuntimeClasspath
-org.postgresql:postgresql:42.7.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.7.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-jdk14:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath

services/tools/gradle.lockfile

@@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath
 org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath
 org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
@@ -338,7 +339,7 @@ org.ow2.asm:asm-tree:9.2=compileClasspath,runtimeClasspath,testCompileClasspath,
 org.ow2.asm:asm-util:7.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.2=compileClasspath,testCompileClasspath
 org.ow2.asm:asm:9.6=runtimeClasspath,testRuntimeClasspath
-org.postgresql:postgresql:42.7.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.postgresql:postgresql:42.7.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-api:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.slf4j:slf4j-jdk14:2.1.0-alpha1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath