Change some READ_COMMITTED levels to REPEATABLE_READ (#1802)

Basically, any time we're loading a bunch of linked objects that might
change, we want to have REPEATABLE_READ so that another transaction
doesn't come along and smush whatever we think we're loading.

The following instances of READ_COMMITTED haven't changed:
- RdePipeline (it only loads immutable objects like histories)
- Invoicing pipeline (only immutable objects like BillingEvents)
- Spec11 (doesn't use any linked info from Domain)

This also changes the PersistenceModule to use REPEATABLE_READ by
default on the replica JPA TM, for the standard reasoning.

config/dependency-license/allowed_licenses.json

@@ -207,6 +207,9 @@
     {
       "moduleLicense": "GNU Library General Public License v2.1 or later"
     },
+    {
+      "moduleLicense": "GNU Lesser General Public License v3.0"
+    },
     // This is just 3-clause BSD.
     {
       "moduleLicense": "Go License"

core/src/main/java/google/registry/beam/resave/ResaveAllEppResourcesPipeline.java

@@ -91,7 +91,6 @@ public class ResaveAllEppResourcesPipeline implements Serializable {
   }
 
   void setupPipeline(Pipeline pipeline) {
-    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     if (options.getFast()) {
       fastResaveContacts(pipeline);
       fastResaveDomains(pipeline);
@@ -194,6 +193,7 @@ public class ResaveAllEppResourcesPipeline implements Serializable {
         PipelineOptionsFactory.fromArgs(args)
             .withValidation()
             .as(ResaveAllEppResourcesPipelineOptions.class);
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
     new ResaveAllEppResourcesPipeline(options).run();
   }
 }

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -102,6 +102,7 @@
     <target>backend</target>
   </cron>
 
+  <!-- TODO(b/249863289): disable until it is safe to run this pipeline
   <cron>
     <url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
     <description>
@@ -110,6 +111,7 @@
     <schedule>1st monday of month 09:00</schedule>
     <target>backend</target>
   </cron>
+  -->
 
   <cron>
     <url><![CDATA[/_dr/task/updateRegistrarRdapBaseUrls]]></url>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -86,6 +86,7 @@
     <target>backend</target>
   </cron>
 
+  <!-- TODO(b/249863289): disable until it is safe to run this pipeline
   <cron>
     <url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
     <description>
@@ -94,6 +95,7 @@
     <schedule>1st monday of month 09:00</schedule>
     <target>backend</target>
   </cron>
+  -->
 
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url>

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -278,7 +278,7 @@ public abstract class PersistenceModule {
     replicaInstanceConnectionName.ifPresent(
         name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
     overrides.put(
-        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
+        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ.name());
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -294,7 +294,7 @@ public abstract class PersistenceModule {
     replicaInstanceConnectionName.ifPresent(
         name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
     overrides.put(
-        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
+        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ.name());
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 

core/src/main/java/google/registry/tools/GetPackagePromotionCommand.java

@@ -0,0 +1,47 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import google.registry.model.domain.token.PackagePromotion;
+import java.util.List;
+
+/** Command to show a {@link PackagePromotion} object. */
+@Parameters(separators = " =", commandDescription = "Show package promotion object(s)")
+public class GetPackagePromotionCommand extends GetEppResourceCommand {
+
+  @Parameter(description = "Package token(s)", required = true)
+  private List<String> mainParameters;
+
+  @Override
+  void runAndPrint() {
+    for (String token : mainParameters) {
+      jpaTm()
+          .transact(
+              () -> {
+                PackagePromotion packagePromotion =
+                    checkArgumentPresent(
+                        PackagePromotion.loadByTokenString(token),
+                        "PackagePromotion with package token %s does not exist",
+                        token);
+                System.out.println(packagePromotion);
+              });
+    }
+  }
+}

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -71,6 +71,7 @@ public final class RegistryTool {
           .put("get_history_entries", GetHistoryEntriesCommand.class)
           .put("get_host", GetHostCommand.class)
           .put("get_keyring_secret", GetKeyringSecretCommand.class)
+          .put("get_package_promotion", GetPackagePromotionCommand.class)
           .put("get_premium_list", GetPremiumListCommand.class)
           .put("get_registrar", GetRegistrarCommand.class)
           .put("get_reserved_list", GetReservedListCommand.class)

core/src/main/java/google/registry/tools/javascrap/CreateSyntheticDomainHistoriesPipeline.java

@@ -60,15 +60,22 @@ public class CreateSyntheticDomainHistoriesPipeline implements Serializable {
   private static final String HISTORY_REASON =
       "Create synthetic domain histories to fix RDE for b/248112997";
   private static final DateTime BAD_PIPELINE_START_TIME =
-      DateTime.parse("2022-09-05T00:00:00.000Z");
+      DateTime.parse("2022-09-05T09:00:00.000Z");
+  private static final DateTime BAD_PIPELINE_END_TIME = DateTime.parse("2022-09-10T12:00:00.000Z");
 
   static void setup(Pipeline pipeline, String registryAdminRegistrarId) {
     pipeline
         .apply(
             "Read all domain repo IDs",
             RegistryJpaIO.read(
-                "SELECT repoId FROM Domain WHERE deletionTime > :badPipelineStartTime",
-                ImmutableMap.of("badPipelineStartTime", BAD_PIPELINE_START_TIME),
+                "SELECT d.repoId FROM Domain d WHERE deletionTime > :badPipelineStartTime AND NOT"
+                    + " EXISTS (SELECT 1 FROM DomainHistory dh WHERE dh.domainRepoId = d.repoId"
+                    + " AND dh.modificationTime > :badPipelineEndTime)",
+                ImmutableMap.of(
+                    "badPipelineStartTime",
+                    BAD_PIPELINE_START_TIME,
+                    "badPipelineEndTime",
+                    BAD_PIPELINE_END_TIME),
                 String.class,
                 repoId -> VKey.createSql(Domain.class, repoId)))
         .apply(
@@ -110,7 +117,7 @@ public class CreateSyntheticDomainHistoriesPipeline implements Serializable {
     RegistryPipelineOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(RegistryPipelineOptions.class);
     RegistryPipelineOptions.validateRegistryPipelineOptions(options);
-    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
     String registryAdminRegistrarId =
         DaggerCreateSyntheticDomainHistoriesPipeline_ConfigComponent.create()
             .getRegistryAdminRegistrarId();

core/src/test/java/google/registry/tools/GetPackagePromotionCommandTest.java

@@ -0,0 +1,129 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import com.beust.jcommander.ParameterException;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.billing.BillingEvent.RenewalPriceBehavior;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.domain.token.AllocationToken.TokenType;
+import google.registry.model.domain.token.PackagePromotion;
+import org.joda.money.CurrencyUnit;
+import org.joda.money.Money;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link GetPackagePromotionCommand}. */
+public class GetPackagePromotionCommandTest extends CommandTestCase<GetPackagePromotionCommand> {
+
+  @Test
+  void testSuccess() throws Exception {
+    AllocationToken token =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("abc123")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    PackagePromotion packagePromotion =
+        new PackagePromotion.Builder()
+            .setToken(token)
+            .setMaxDomains(100)
+            .setMaxCreates(500)
+            .setPackagePrice(Money.of(CurrencyUnit.USD, 1000))
+            .setNextBillingDate(DateTime.parse("2012-11-12T05:00:00Z"))
+            .setLastNotificationSent(DateTime.parse("2010-11-12T05:00:00Z"))
+            .build();
+    jpaTm().transact(() -> jpaTm().put(packagePromotion));
+    runCommand("abc123");
+  }
+
+  @Test
+  void testSuccessMultiplePackages() throws Exception {
+    AllocationToken token =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("abc123")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .put(
+                        new PackagePromotion.Builder()
+                            .setToken(token)
+                            .setMaxDomains(100)
+                            .setMaxCreates(500)
+                            .setPackagePrice(Money.of(CurrencyUnit.USD, 1000))
+                            .setNextBillingDate(DateTime.parse("2012-11-12T05:00:00Z"))
+                            .setLastNotificationSent(DateTime.parse("2010-11-12T05:00:00Z"))
+                            .build()));
+    AllocationToken token2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("123abc")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2012-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .put(
+                        new PackagePromotion.Builder()
+                            .setToken(token2)
+                            .setMaxDomains(1000)
+                            .setMaxCreates(700)
+                            .setPackagePrice(Money.of(CurrencyUnit.USD, 3000))
+                            .setNextBillingDate(DateTime.parse("2014-11-12T05:00:00Z"))
+                            .setLastNotificationSent(DateTime.parse("2013-11-12T05:00:00Z"))
+                            .build()));
+
+    runCommand("abc123", "123abc");
+  }
+
+  @Test
+  void testFailure_packageDoesNotExist() {
+    IllegalArgumentException thrown =
+        assertThrows(IllegalArgumentException.class, () -> runCommand("fakeToken"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo("PackagePromotion with package token fakeToken does not exist");
+  }
+
+  @Test
+  void testFailure_noToken() {
+    assertThrows(ParameterException.class, this::runCommand);
+  }
+}

core/src/test/java/google/registry/tools/javascrap/CreateSyntheticDomainHistoriesPipelineTest.java

@@ -16,21 +16,24 @@ package google.registry.tools.javascrap;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.loadAllOf;
-import static google.registry.testing.DatabaseHelper.newDomain;
+import static google.registry.testing.DatabaseHelper.newContact;
 import static google.registry.testing.DatabaseHelper.persistActiveHost;
+import static google.registry.testing.DatabaseHelper.persistDomainWithDependentResources;
 import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
+import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.DatabaseHelper.persistSimpleResource;
 
-import com.google.common.collect.Iterables;
 import google.registry.beam.TestPipelineExtension;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.persistence.transaction.JpaTestExtensions;
 import google.registry.testing.DatastoreEntityExtension;
+import google.registry.testing.FakeClock;
+import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -38,9 +41,11 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 /** Tests for {@link CreateSyntheticDomainHistoriesPipeline}. */
 public class CreateSyntheticDomainHistoriesPipelineTest {
 
+  private final FakeClock fakeClock = new FakeClock(DateTime.parse("2022-09-01T00:00:00.000Z"));
+
   @RegisterExtension
   JpaTestExtensions.JpaIntegrationTestExtension jpaEextension =
-      new JpaTestExtensions.Builder().buildIntegrationTestExtension();
+      new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationTestExtension();
 
   @RegisterExtension
   DatastoreEntityExtension datastoreEntityExtension =
@@ -55,24 +60,49 @@ public class CreateSyntheticDomainHistoriesPipelineTest {
     persistNewRegistrar("TheRegistrar");
     persistNewRegistrar("NewRegistrar");
     createTld("tld");
+    domain =
+        persistDomainWithDependentResources(
+            "example",
+            "tld",
+            persistResource(newContact("contact1234")),
+            fakeClock.nowUtc(),
+            DateTime.parse("2022-09-01T00:00:00.000Z"),
+            DateTime.parse("2024-09-01T00:00:00.000Z"));
     domain =
         persistSimpleResource(
-            newDomain("example.tld")
+            domain
                 .asBuilder()
                 .setNameservers(persistActiveHost("external.com").createVKey())
                 .build());
+    fakeClock.setTo(DateTime.parse("2022-09-20T00:00:00.000Z"));
+    // shouldn't create any history objects for this domain
+    persistDomainWithDependentResources(
+        "ignored-example",
+        "tld",
+        persistResource(newContact("contact12345")),
+        fakeClock.nowUtc(),
+        DateTime.parse("2022-09-20T00:00:00.000Z"),
+        DateTime.parse("2024-09-20T00:00:00.000Z"));
   }
 
   @Test
   void testSuccess() {
-    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(DomainHistory.class))).isEmpty();
+    assertThat(loadAllOf(DomainHistory.class)).hasSize(2);
     CreateSyntheticDomainHistoriesPipeline.setup(pipeline, "NewRegistrar");
     pipeline.run().waitUntilFinish();
-    DomainHistory domainHistory = Iterables.getOnlyElement(loadAllOf(DomainHistory.class));
-    assertThat(domainHistory.getType()).isEqualTo(HistoryEntry.Type.SYNTHETIC);
-    assertThat(domainHistory.getRegistrarId()).isEqualTo("NewRegistrar");
+    DomainHistory syntheticHistory =
+        HistoryEntryDao.loadHistoryObjectsForResource(domain.createVKey(), DomainHistory.class)
+            .get(1);
+    assertThat(syntheticHistory.getType()).isEqualTo(HistoryEntry.Type.SYNTHETIC);
+    assertThat(syntheticHistory.getRegistrarId()).isEqualTo("NewRegistrar");
     assertAboutImmutableObjects()
-        .that(domainHistory.getDomainBase().get())
-        .hasFieldsEqualTo(domain);
+        .that(syntheticHistory.getDomainBase().get())
+        .isEqualExceptFields(domain, "updateTimestamp");
+
+    // shouldn't create any entries on re-run
+    pipeline.run().waitUntilFinish();
+    assertThat(HistoryEntryDao.loadHistoryObjectsForResource(domain.createVKey())).hasSize(2);
+    // three total histories, two CREATE and one SYNTHETIC
+    assertThat(loadAllOf(DomainHistory.class)).hasSize(3);
   }
 }

java_common.gradle

@@ -62,6 +62,12 @@ configurations {
     // See https://issues.apache.org/jira/browse/BEAM-8862
     it.exclude group: 'org.mockito', module: 'mockito-core'
   }
+  all.each {
+    // log4j has high-profile security vulnerabilities. It's a transitive
+    // dependency used by some Apache Beam packages. Excluding it does not
+    // impact our troubleshooting needs.
+    it.exclude group: 'org.apache.logging.log4j'
+  }
 }
 
 dependencies {