Add a new recurrenceLastExpansion column to the BillingRecurrence table (#1650)

* Add a new recurrenceLastExpansion column to the BillingRecurrence table

This will be used to determine which recurrences are in scope for
expansion. Anything that has already been expanded within the past year is
automatically out of scope.

Note that the default value is set to just over a year ago, so that, initially,
everything is in scope for expansion, and then will gradually be winnowed down
over time so that most recurrences at any given point are out of scope. Newly
created recurrings (after the subsequent code change goes in) will have their
last expansion time set to the same as the event time of when the recurring is
written, such that they'll first be considered in-scope precisely one year
later.

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -44,6 +44,7 @@ import com.google.common.collect.Range;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.flows.domain.DomainPricingLogic;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.NullInput;
 import google.registry.model.ImmutableObject;
@@ -98,6 +99,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
 
   @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
   @Inject @Parameter(PARAM_CURSOR_TIME) Optional<DateTime> cursorTimeParam;
+
+  @Inject DomainPricingLogic domainPricingLogic;
   @Inject Response response;
   @Inject ExpandRecurringBillingEventsAction() {}
 

core/src/main/java/google/registry/flows/custom/BaseFlowCustomLogic.java

@@ -17,6 +17,8 @@ package google.registry.flows.custom;
 import google.registry.flows.FlowMetadata;
 import google.registry.flows.SessionMetadata;
 import google.registry.model.eppinput.EppInput;
+import java.util.Optional;
+import javax.annotation.Nullable;
 
 /**
  * An abstract base class for all flow custom logic that stores the flow's {@link EppInput} and
@@ -24,26 +26,38 @@ import google.registry.model.eppinput.EppInput;
  */
 public abstract class BaseFlowCustomLogic {
 
-  private final EppInput eppInput;
-  private final SessionMetadata sessionMetadata;
-  private final FlowMetadata flowMetadata;
+  @Nullable private final EppInput eppInput;
+  @Nullable private final SessionMetadata sessionMetadata;
+  @Nullable private final FlowMetadata flowMetadata;
 
+  /**
+   * Constructs a BaseFlowCustomLogic for the specified EPP flow state.
+   *
+   * <p>Note that it is possible for the EPP flow state to be absent, which happens when the custom
+   * logic is running outside the context of an EPP flow (e.g. {@link DomainPricingCustomLogic} in
+   * backend actions).
+   */
   protected BaseFlowCustomLogic(
-      EppInput eppInput, SessionMetadata sessionMetadata, FlowMetadata flowMetadata) {
+      @Nullable EppInput eppInput,
+      @Nullable SessionMetadata sessionMetadata,
+      @Nullable FlowMetadata flowMetadata) {
     this.eppInput = eppInput;
     this.sessionMetadata = sessionMetadata;
     this.flowMetadata = flowMetadata;
   }
 
-  protected EppInput getEppInput() {
-    return eppInput;
+  /** Returns the {@link EppInput}, which may be empty outside a flow context. */
+  protected Optional<EppInput> getEppInput() {
+    return Optional.ofNullable(eppInput);
   }
 
-  protected SessionMetadata getSessionMetadata() {
-    return sessionMetadata;
+  /** Returns the {@link SessionMetadata}, which may be empty outside a flow context. */
+  protected Optional<SessionMetadata> getSessionMetadata() {
+    return Optional.ofNullable(sessionMetadata);
   }
 
-  protected FlowMetadata getFlowMetadata() {
-    return flowMetadata;
+  /** Returns the {@link FlowMetadata}, which may be empty outside a flow context. */
+  protected Optional<FlowMetadata> getFlowMetadata() {
+    return Optional.ofNullable(flowMetadata);
   }
 }

core/src/main/java/google/registry/flows/custom/CustomLogicFactory.java

@@ -18,6 +18,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.flows.FlowMetadata;
 import google.registry.flows.SessionMetadata;
 import google.registry.model.eppinput.EppInput;
+import java.util.Optional;
 
 /**
  * A no-op base custom logic factory.
@@ -63,7 +64,10 @@ public class CustomLogicFactory {
   }
 
   public DomainPricingCustomLogic forDomainPricing(
-      EppInput eppInput, SessionMetadata sessionMetadata, FlowMetadata flowMetadata) {
-    return new DomainPricingCustomLogic(eppInput, sessionMetadata, flowMetadata);
+      Optional<EppInput> eppInput,
+      Optional<SessionMetadata> sessionMetadata,
+      Optional<FlowMetadata> flowMetadata) {
+    return new DomainPricingCustomLogic(
+        eppInput.orElse(null), sessionMetadata.orElse(null), flowMetadata.orElse(null));
   }
 }

core/src/main/java/google/registry/flows/custom/CustomLogicModule.java

@@ -14,15 +14,17 @@
 
 package google.registry.flows.custom;
 
+import dagger.BindsOptionalOf;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.flows.FlowMetadata;
 import google.registry.flows.SessionMetadata;
 import google.registry.model.eppinput.EppInput;
+import java.util.Optional;
 
 /** Dagger module to provide instances of custom logic classes for EPP flows. */
 @Module
-public class CustomLogicModule {
+public abstract class CustomLogicModule {
 
   @Provides
   static DomainCreateFlowCustomLogic provideDomainCreateFlowCustomLogic(
@@ -81,9 +83,20 @@ public class CustomLogicModule {
   @Provides
   static DomainPricingCustomLogic provideDomainPricingCustomLogic(
       CustomLogicFactory factory,
-      EppInput eppInput,
-      SessionMetadata sessionMetadata,
-      FlowMetadata flowMetadata) {
+      Optional<EppInput> eppInput,
+      Optional<SessionMetadata> sessionMetadata,
+      Optional<FlowMetadata> flowMetadata) {
+    // Note that, for DomainPricingCustomLogic, the EPP flow state won't be present outside the
+    // context of an EPP flow.
     return factory.forDomainPricing(eppInput, sessionMetadata, flowMetadata);
   }
+
+  @BindsOptionalOf
+  abstract EppInput optionalEppInput();
+
+  @BindsOptionalOf
+  abstract SessionMetadata optionalSessionMetadata();
+
+  @BindsOptionalOf
+  abstract FlowMetadata optionalFlowMetadata();
 }

core/src/main/java/google/registry/flows/custom/DomainPricingCustomLogic.java

@@ -24,6 +24,7 @@ import google.registry.flows.domain.FeesAndCredits;
 import google.registry.model.ImmutableObject;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.tld.Registry;
+import javax.annotation.Nullable;
 import org.joda.time.DateTime;
 
 /**
@@ -34,7 +35,9 @@ import org.joda.time.DateTime;
 public class DomainPricingCustomLogic extends BaseFlowCustomLogic {
 
   public DomainPricingCustomLogic(
-      EppInput eppInput, SessionMetadata sessionMetadata, FlowMetadata flowMetadata) {
+      @Nullable EppInput eppInput,
+      @Nullable SessionMetadata sessionMetadata,
+      @Nullable FlowMetadata flowMetadata) {
     super(eppInput, sessionMetadata, flowMetadata);
   }
 

core/src/main/java/google/registry/flows/domain/DomainPricingLogic.java

@@ -23,7 +23,6 @@ import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
 import com.google.common.net.InternetDomainName;
 import google.registry.flows.EppException;
 import google.registry.flows.EppException.CommandUseErrorException;
-import google.registry.flows.FlowScope;
 import google.registry.flows.custom.DomainPricingCustomLogic;
 import google.registry.flows.custom.DomainPricingCustomLogic.CreatePriceParameters;
 import google.registry.flows.custom.DomainPricingCustomLogic.RenewPriceParameters;
@@ -50,7 +49,6 @@ import org.joda.time.DateTime;
  * providing a {@link DomainPricingCustomLogic} implementation that operates on cross-TLD or per-TLD
  * logic.
  */
-@FlowScope
 public final class DomainPricingLogic {
 
   @Inject DomainPricingCustomLogic customLogic;

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -61,6 +61,7 @@ import google.registry.export.UploadDatastoreBackupAction;
 import google.registry.export.sheet.SheetModule;
 import google.registry.export.sheet.SyncRegistrarsSheetAction;
 import google.registry.flows.FlowComponent;
+import google.registry.flows.custom.CustomLogicModule;
 import google.registry.mapreduce.MapreduceModule;
 import google.registry.model.replay.ReplicateToDatastoreAction;
 import google.registry.monitoring.whitebox.WhiteboxModule;
@@ -103,6 +104,7 @@ import google.registry.tools.javascrap.CreateSyntheticHistoryEntriesAction;
       BillingModule.class,
       CloudDnsWriterModule.class,
       CronModule.class,
+      CustomLogicModule.class,
       DnsCountQueryCoordinatorModule.class,
       DnsModule.class,
       DnsUpdateConfigModule.class,

core/src/main/java/google/registry/module/tools/ToolsRequestComponent.java

@@ -48,14 +48,14 @@ import google.registry.tools.server.VerifyOteAction;
 @RequestScope
 @Subcomponent(
     modules = {
-        BackupModule.class,
-        DnsModule.class,
-        EppToolModule.class,
-        LoadTestModule.class,
-        MapreduceModule.class,
-        RequestModule.class,
-        ToolsServerModule.class,
-        WhiteboxModule.class,
+      BackupModule.class,
+      DnsModule.class,
+      EppToolModule.class,
+      LoadTestModule.class,
+      MapreduceModule.class,
+      RequestModule.class,
+      ToolsServerModule.class,
+      WhiteboxModule.class,
     })
 interface ToolsRequestComponent {
   CreateGroupsAction createGroupsAction();

core/src/test/java/google/registry/flows/custom/TestCustomLogicFactory.java

@@ -17,6 +17,7 @@ package google.registry.flows.custom;
 import google.registry.flows.FlowMetadata;
 import google.registry.flows.SessionMetadata;
 import google.registry.model.eppinput.EppInput;
+import java.util.Optional;
 
 /** A custom logic factory for testing. */
 public class TestCustomLogicFactory extends CustomLogicFactory {
@@ -29,7 +30,10 @@ public class TestCustomLogicFactory extends CustomLogicFactory {
 
   @Override
   public DomainPricingCustomLogic forDomainPricing(
-      EppInput eppInput, SessionMetadata sessionMetadata, FlowMetadata flowMetadata) {
-    return new TestDomainPricingCustomLogic(eppInput, sessionMetadata, flowMetadata);
+      Optional<EppInput> eppInput,
+      Optional<SessionMetadata> sessionMetadata,
+      Optional<FlowMetadata> flowMetadata) {
+    return new TestDomainPricingCustomLogic(
+        eppInput.orElse(null), sessionMetadata.orElse(null), flowMetadata.orElse(null));
   }
 }

core/src/test/java/google/registry/flows/custom/TestDomainCreateFlowCustomLogic.java

@@ -36,7 +36,7 @@ public class TestDomainCreateFlowCustomLogic extends DomainCreateFlowCustomLogic
           new PollMessage.OneTime.Builder()
               .setParent(parameters.historyEntry())
               .setEventTime(tm().getTransactionTime())
-              .setRegistrarId(getSessionMetadata().getRegistrarId())
+              .setRegistrarId(getSessionMetadata().get().getRegistrarId())
               .setMsg("Custom logic was triggered")
               .build();
       return EntityChanges.newBuilder()

db/README.md

@@ -120,6 +120,72 @@ update the Java to no longer contain the old column, wait for a deployment, and
 then remove the old column. A rename operation requires the most complicated
 series of steps to complete, as it is effectively an add followed by a remove.
 
+### Summary of Schema Tests
+
+#### The Golden Schema Test
+
+The ":db:test" task runs a task that verifies that the database schema as
+specified by the entire set of Flyway scripts is valid and matches
+'nomulus.golden.sql'.
+
+As mentioned in the previous section, you may run
+`./nom_build :nom:generate_golden_file` to update the golden schema.
+
+#### The Forbidden Flyway Script Change Detection Test
+
+Once a Flyway DDL script is deployed to Sandbox or Production, it must not be
+changed. During each schema deployment, Flyway checks all past scripts against
+its record, and aborts if any of them do not match.
+
+This test is not part of the local Gradle build. It is part of the presubmit
+tests for the FOSS repo.
+
+To test locally, run `./integration/run_schema_check -p domain-registry-dev`
+from the root directory of the Nomulus repo.
+
+#### The Server-Schema Compatibility Test
+
+This test ensures that the Nomulus server code in the current branch is
+compatible with the deployed schemas in Sandbox and Production; and that the
+schema change to be submitted is compatible with the Nomulus servers currently
+deployed to Sandbox and Production. Note that this test fetches schemas packaged
+in the appropriate release artifacts, not from the live database.
+
+This test is not part of the local Gradle build. It is part of the presubmit
+tests for the FOSS repo.
+
+To test locally, run the following commands from the root directory of the
+Nomulus repo:
+
+```shell
+$ git fetch --tags
+# Following command tests local Java code against released schemas
+$ ./integration/run_compatibility_tests -p domain-registry-dev \
+    -s nomulus
+# Following command tests deployed code against local schema
+$ ./integration/run_compatibility_tests -p domain-registry-dev \
+    -s sql
+```
+
+#### The Out-Of-Band Schema Change Test
+
+This test verifies that the actual schema from the live database in Sandbox or
+Production matches the golden schema. It detects changes made by, e.g.,
+operators during troubleshooting.
+
+This test is part of the Spinnaker deployment pipelines for Sandbox and
+Production. It is the first step in the pipeline, and halts the pipeline if the
+test fails. This is advantageous to testing in the last step of the pipeline,
+where failures sometimes escaped notice.
+
+To run this locally, run the following commands from the root directory of the
+Nomulus repo:
+
+```shell
+$ (cd release; gcloud builds submit --config=cloudbuild-schema-verify.yaml \
+  --substitutions=_ENV=[sandbox|production] ..)
+```
+
 ### Schema push
 
 Currently Cloud SQL schema is released with the Nomulus server, and shares the

db/src/main/resources/sql/flyway.txt

@@ -114,3 +114,4 @@ V113__add_host_missing_indexes.sql
 V114__add_allocation_token_indexes.sql
 V115__add_renewal_columns_to_billing_recurrence.sql
 V116__add_renewal_column_to_allocation_token.sql
+V117__add_billing_recurrence_last_expansion_column.sql

db/src/main/resources/sql/flyway/V117__add_billing_recurrence_last_expansion_column.sql

@@ -0,0 +1,16 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "BillingRecurrence" add column recurrence_last_expansion timestamptz default '2021-06-01T00:00:00Z' not null;
+create index IDXp0pxi708hlu4n40qhbtihge8x on "BillingRecurrence" (recurrence_last_expansion);

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -120,7 +120,8 @@ CREATE TABLE public."BillingRecurrence" (
     recurrence_time_of_year text,
     renewal_price_behavior text DEFAULT 'DEFAULT'::text NOT NULL,
     renewal_price_currency text,
-    renewal_price_amount numeric(19,2)
+    renewal_price_amount numeric(19,2),
+    recurrence_last_expansion timestamp with time zone DEFAULT '2021-06-01 00:00:00+00'::timestamp with time zone NOT NULL
 );
 
 
@@ -1879,6 +1880,13 @@ CREATE INDEX idxoqttafcywwdn41um6kwlt0n8b ON public."BillingRecurrence" USING bt
 CREATE INDEX idxovmntef6l45tw2bsfl56tcugx ON public."Host" USING btree (deletion_time);
 
 
+--
+-- Name: idxp0pxi708hlu4n40qhbtihge8x; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxp0pxi708hlu4n40qhbtihge8x ON public."BillingRecurrence" USING btree (recurrence_last_expansion);
+
+
 --
 -- Name: idxp3usbtvk0v1m14i5tdp4xnxgc; Type: INDEX; Schema: public; Owner: -
 --