Does not self allocate IDs in Beam by default. (#1809)

* Does not self allocate IDs in Beam by default.

Per b/250948425, it is dangerous to implicitly allow all Beam pipelines
to create buildables by self allocating the IDs. This change makes it so
that one has to explicitly request self allocation in Beam.

A boolean is added to the pipeline option so that it can be passed to
the beam worker initializer that controls the behavior of the JVM on
each worker. Note that we did not add the option in the metadata.json file
because we did not want people to use the override at run time when launching
a pipeline, due to the risk. As shown in RdePipeline.java, we instead
explicitly hard-code the option in the pipeline. There is nothing that
stops one to supply that option when launching the pipeline, but it's
not advised.

Tested=deployed the pipeline alpha and ran it.

config/dependency-license/allowed_licenses.json

@@ -207,6 +207,9 @@
     {
       "moduleLicense": "GNU Library General Public License v2.1 or later"
     },
+    {
+      "moduleLicense": "GNU Lesser General Public License v3.0"
+    },
     // This is just 3-clause BSD.
     {
       "moduleLicense": "Go License"

core/src/main/java/google/registry/batch/CheckPackagesComplianceAction.java

@@ -0,0 +1,80 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package google.registry.batch;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.token.PackagePromotion;
+import google.registry.request.Action;
+import google.registry.request.Action.Service;
+import google.registry.request.auth.Auth;
+import java.util.List;
+
+/**
+ * An action that checks all {@link PackagePromotion} objects for compliance with their max create
+ * limit.
+ */
+@Action(
+    service = Service.BACKEND,
+    path = CheckPackagesComplianceAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class CheckPackagesComplianceAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/checkPackagesCompliance";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  @Override
+  public void run() {
+    tm().transact(
+            () -> {
+              ImmutableList<PackagePromotion> packages = tm().loadAllOf(PackagePromotion.class);
+              ImmutableList.Builder<PackagePromotion> packagesOverCreateLimit =
+                  new ImmutableList.Builder<>();
+              for (PackagePromotion packagePromo : packages) {
+                List<DomainHistory> creates =
+                    jpaTm()
+                        .query(
+                            "FROM DomainHistory WHERE current_package_token = :token AND"
+                                + " modificationTime >= :lastBilling AND type = 'DOMAIN_CREATE'",
+                            DomainHistory.class)
+                        .setParameter("token", packagePromo.getToken().getSqlKey().toString())
+                        .setParameter(
+                            "lastBilling", packagePromo.getNextBillingDate().minusYears(1))
+                        .getResultList();
+
+                if (creates.size() > packagePromo.getMaxCreates()) {
+                  int overage = creates.size() - packagePromo.getMaxCreates();
+                  logger.atInfo().log(
+                      "Package with package token %s has exceeded their max domain creation limit"
+                          + " by %d name(s).",
+                      packagePromo.getToken().getSqlKey(), overage);
+                  packagesOverCreateLimit.add(packagePromo);
+                }
+              }
+              if (packagesOverCreateLimit.build().isEmpty()) {
+                logger.atInfo().log("Found no packages over their create limit.");
+              } else {
+                logger.atInfo().log(
+                    "Found %d packages over their create limit.",
+                    packagesOverCreateLimit.build().size());
+                // TODO(sarahbot@) Send email to registrar and registry informing of creation
+                // overage once email template is finalized.
+              }
+            });
+  }
+}

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
@@ -65,6 +66,17 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setSqlWriteShards(int maxConcurrentSqlWriters);
 
+  @DeleteAfterMigration
+  @Description(
+      "Whether to use self allocated primary IDs when building entities. This should only be used"
+          + " when the IDs are not significant and the resulting entities are not persisted back to"
+          + " the database. Use with caution as self allocated IDs are not unique across workers,"
+          + " and persisting entities with these IDs can be dangerous.")
+  @Default.Boolean(false)
+  boolean getUseSelfAllocatedId();
+
+  void setUseSelfAllocatedId(boolean useSelfAllocatedId);
+
   static RegistryPipelineComponent toRegistryPipelineComponent(RegistryPipelineOptions options) {
     return DaggerRegistryPipelineComponent.builder()
         .isolationOverride(options.getIsolationOverride())

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -22,6 +22,8 @@ import dagger.Lazy;
 import google.registry.config.RegistryEnvironment;
 import google.registry.config.SystemPropertySetter;
 import google.registry.model.AppEngineEnvironment;
+import google.registry.model.IdService;
+import google.registry.model.IdService.SelfAllocatedIdSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import org.apache.beam.sdk.harness.JvmInitializer;
@@ -65,12 +67,20 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
         transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
     }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
-    // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
+    // Masquerade all threads as App Engine threads, so we can create Ofy keys in the pipeline. Also
     // loads all ofy entities.
     new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
         .setEnvironmentForAllThreads();
-    // Set the system property so that we can call IdService.allocateId() without access to
-    // datastore.
     SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");
+    // Use self-allocated IDs if requested. Note that this inevitably results in duplicate IDs from
+    // multiple workers, which can also collide with existing IDs in the database. So they cannot be
+    // dependent upon for comparison or anything significant. The resulting entities can never be
+    // persisted back into the database. This is a stop-gap measure that should only be used when
+    // you need to create Buildables in Beam, but do not have control over how the IDs are
+    // allocated, and you don't care about the generated IDs as long
+    // as you can build the entities.
+    if (registryOptions.getUseSelfAllocatedId()) {
+      IdService.setIdSupplier(SelfAllocatedIdSupplier.getInstance());
+    }
   }
 }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -128,7 +128,7 @@ import org.joda.time.DateTime;
  * <h2>{@link EppResource}</h2>
  *
  * All EPP resources are loaded from the corresponding {@link HistoryEntry}, which has the resource
- * embedded. In general we find most recent history entry before watermark and filter out the ones
+ * embedded. In general, we find most recent history entry before watermark and filter out the ones
  * that are soft-deleted by watermark. The history is emitted as pairs of (resource repo ID: history
  * revision ID) from the SQL query.
  *
@@ -164,7 +164,7 @@ import org.joda.time.DateTime;
  *
  * The (pending deposit: deposit fragment) pairs from different resources are combined and grouped
  * by pending deposit. For each pending deposit, all the relevant deposit fragments are written into
- * a encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
+ * an encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
  * is no need to lock the GCS write operation to prevent stomping. The cursor for staging the
  * pending deposit is then rolled forward, and the next action is enqueued. The latter two
  * operations are performed in a transaction so the cursor is rolled back if enqueueing failed.
@@ -698,8 +698,8 @@ public class RdePipeline implements Serializable {
   }
 
   /**
-   * Encodes the pending deposit set in an URL safe string that is sent to the pipeline worker by
-   * the pipeline launcher as a pipeline option.
+   * Encodes the pending deposit set in a URL safe string that is sent to the pipeline worker by the
+   * pipeline launcher as a pipeline option.
    */
   public static String encodePendingDeposits(ImmutableSet<PendingDeposit> pendingDeposits)
       throws IOException {
@@ -715,6 +715,12 @@ public class RdePipeline implements Serializable {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
     RdePipelineOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
+    // We need to self allocate the IDs because the pipeline creates EPP resources from history
+    // entries and projects them to watermark. These buildable entities would otherwise request an
+    // ID from datastore, which Beam does not have access to. The IDs are not included in the
+    // deposits or are these entities persisted back to the database, so it is OK to use a self
+    // allocated ID to get around the limitations of beam.
+    options.setUseSelfAllocatedId(true);
     RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();

core/src/main/java/google/registry/beam/resave/ResaveAllEppResourcesPipeline.java

@@ -91,7 +91,6 @@ public class ResaveAllEppResourcesPipeline implements Serializable {
   }
 
   void setupPipeline(Pipeline pipeline) {
-    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     if (options.getFast()) {
       fastResaveContacts(pipeline);
       fastResaveDomains(pipeline);
@@ -194,6 +193,7 @@ public class ResaveAllEppResourcesPipeline implements Serializable {
         PipelineOptionsFactory.fromArgs(args)
             .withValidation()
             .as(ResaveAllEppResourcesPipelineOptions.class);
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
     new ResaveAllEppResourcesPipeline(options).run();
   }
 }

core/src/main/java/google/registry/beam/spec11/ThreatMatch.java

@@ -26,6 +26,7 @@ public abstract class ThreatMatch implements Serializable {
 
   private static final String THREAT_TYPE_FIELD = "threatType";
   private static final String DOMAIN_NAME_FIELD = "domainName";
+  private static final String OUTDATED_NAME_FIELD = "fullyQualifiedDomainName";
 
   /** Returns what kind of threat it is (malware, phishing etc.) */
   public abstract String threatType();
@@ -46,7 +47,12 @@ public abstract class ThreatMatch implements Serializable {
 
   /** Parses a {@link JSONObject} and returns an equivalent {@link ThreatMatch}. */
   public static ThreatMatch fromJSON(JSONObject threatMatch) throws JSONException {
+    // TODO: delete OUTDATED_NAME_FIELD once we no longer process reports saved with
+    // fullyQualifiedDomainName in them, likely 2023
     return new AutoValue_ThreatMatch(
-        threatMatch.getString(THREAT_TYPE_FIELD), threatMatch.getString(DOMAIN_NAME_FIELD));
+        threatMatch.getString(THREAT_TYPE_FIELD),
+        threatMatch.has(OUTDATED_NAME_FIELD)
+            ? threatMatch.getString(OUTDATED_NAME_FIELD)
+            : threatMatch.getString(DOMAIN_NAME_FIELD));
   }
 }

core/src/main/java/google/registry/config/CredentialModule.java

@@ -16,7 +16,6 @@ package google.registry.config;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.common.collect.ImmutableList;
 import dagger.Module;
@@ -37,6 +36,36 @@ import javax.inject.Singleton;
 @Module
 public abstract class CredentialModule {
 
+  /**
+   * Provides a {@link GoogleCredentialsBundle} backed by the application default credential from
+   * the Google Cloud Runtime. This credential may be used to access GCP APIs that are NOT part of
+   * the Google Workspace.
+   *
+   * <p>The credential returned by the Cloud Runtime depends on the runtime environment:
+   *
+   * <ul>
+   *   <li>On App Engine, returns a scope-less {@code ComputeEngineCredentials} for
+   *       PROJECT_ID@appspot.gserviceaccount.com
+   *   <li>On Compute Engine, returns a scope-less {@code ComputeEngineCredentials} for
+   *       PROJECT_NUMBER-compute@developer.gserviceaccount.com
+   *   <li>On end user host, this returns the credential downloaded by gcloud. Please refer to <a
+   *       href="https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login">Cloud
+   *       SDK documentation</a> for details.
+   * </ul>
+   */
+  @ApplicationDefaultCredential
+  @Provides
+  @Singleton
+  public static GoogleCredentialsBundle provideApplicationDefaultCredential() {
+    GoogleCredentials credential;
+    try {
+      credential = GoogleCredentials.getApplicationDefault();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return GoogleCredentialsBundle.create(credential);
+  }
+
   /**
    * Provides the default {@link GoogleCredentialsBundle} from the Google Cloud runtime.
    *
@@ -70,26 +99,19 @@ public abstract class CredentialModule {
   }
 
   /**
-   * Provides the default {@link GoogleCredential} from the Google Cloud runtime for G Suite
-   * Drive API.
-   * TODO(b/138195359): Deprecate this credential once we figure out how to use
-   * {@link GoogleCredentials} for G Suite Drive API.
+   * Provides a {@link GoogleCredentialsBundle} for accessing Google Workspace APIs, such as Drive
+   * and Sheets.
    */
-  @GSuiteDriveCredential
+  @GoogleWorkspaceCredential
   @Provides
   @Singleton
-  public static GoogleCredential provideGSuiteDriveCredential(
+  public static GoogleCredentialsBundle provideGSuiteDriveCredential(
+      @ApplicationDefaultCredential GoogleCredentialsBundle applicationDefaultCredential,
       @Config("defaultCredentialOauthScopes") ImmutableList<String> requiredScopes) {
-    GoogleCredential credential;
-    try {
-      credential = GoogleCredential.getApplicationDefault();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-    if (credential.createScopedRequired()) {
-      credential = credential.createScoped(requiredScopes);
-    }
-    return credential;
+    GoogleCredentials credential = applicationDefaultCredential.getGoogleCredentials();
+    // Although credential is scope-less, its `createScopedRequired` method still returns false.
+    credential = credential.createScoped(requiredScopes);
+    return GoogleCredentialsBundle.create(credential);
   }
 
   /**
@@ -136,18 +158,24 @@ public abstract class CredentialModule {
         .createScoped(requiredScopes));
   }
 
+  /** Dagger qualifier for the scope-less Application Default Credential. */
+  @Qualifier
+  @Documented
+  @Retention(RetentionPolicy.RUNTIME)
+  public @interface ApplicationDefaultCredential {}
+
   /** Dagger qualifier for the Application Default Credential. */
   @Qualifier
   @Documented
   @Retention(RetentionPolicy.RUNTIME)
+  @Deprecated // Switching to @ApplicationDefaultCredential
   public @interface DefaultCredential {}
 
-
-  /** Dagger qualifier for the credential for G Suite Drive API. */
+  /** Dagger qualifier for the credential for Google Workspace APIs. */
   @Qualifier
   @Documented
   @Retention(RetentionPolicy.RUNTIME)
-  public @interface GSuiteDriveCredential {}
+  public @interface GoogleWorkspaceCredential {}
 
   /**
    * Dagger qualifier for a credential from a service account's JSON key, to be used in non-request

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -102,6 +102,7 @@
     <target>backend</target>
   </cron>
 
+  <!-- TODO(b/249863289): disable until it is safe to run this pipeline
   <cron>
     <url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
     <description>
@@ -110,6 +111,7 @@
     <schedule>1st monday of month 09:00</schedule>
     <target>backend</target>
   </cron>
+  -->
 
   <cron>
     <url><![CDATA[/_dr/task/updateRegistrarRdapBaseUrls]]></url>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -86,6 +86,7 @@
     <target>backend</target>
   </cron>
 
+  <!-- TODO(b/249863289): disable until it is safe to run this pipeline
   <cron>
     <url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
     <description>
@@ -94,6 +95,7 @@
     <schedule>1st monday of month 09:00</schedule>
     <target>backend</target>
   </cron>
+  -->
 
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url>

core/src/main/java/google/registry/export/DriveModule.java

@@ -14,16 +14,16 @@
 
 package google.registry.export;
 
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.services.drive.Drive;
 import dagger.Component;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.CredentialModule;
-import google.registry.config.CredentialModule.GSuiteDriveCredential;
+import google.registry.config.CredentialModule.GoogleWorkspaceCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.storage.drive.DriveConnection;
+import google.registry.util.GoogleCredentialsBundle;
 import javax.inject.Singleton;
 
 /** Dagger module for Google {@link Drive} service connection objects. */
@@ -32,13 +32,13 @@ public final class DriveModule {
 
   @Provides
   static Drive provideDrive(
-      @GSuiteDriveCredential GoogleCredential googleCredential,
+      @GoogleWorkspaceCredential GoogleCredentialsBundle googleCredential,
       @Config("projectId") String projectId) {
 
     return new Drive.Builder(
-            googleCredential.getTransport(),
+            googleCredential.getHttpTransport(),
             googleCredential.getJsonFactory(),
-            googleCredential)
+            googleCredential.getHttpRequestInitializer())
         .setApplicationName(projectId)
         .build();
   }

core/src/main/java/google/registry/export/sheet/SheetsServiceModule.java

@@ -17,7 +17,7 @@ package google.registry.export.sheet;
 import com.google.api.services.sheets.v4.Sheets;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.config.CredentialModule.JsonCredential;
+import google.registry.config.CredentialModule.GoogleWorkspaceCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.util.GoogleCredentialsBundle;
 
@@ -27,7 +27,7 @@ public final class SheetsServiceModule {
 
   @Provides
   static Sheets provideSheets(
-      @JsonCredential GoogleCredentialsBundle credentialsBundle,
+      @GoogleWorkspaceCredential GoogleCredentialsBundle credentialsBundle,
       @Config("projectId") String projectId) {
     return new Sheets.Builder(
             credentialsBundle.getHttpTransport(),

core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java

@@ -32,8 +32,6 @@ import static google.registry.flows.domain.DomainTransferUtils.createLosingTrans
 import static google.registry.flows.domain.DomainTransferUtils.createPendingTransferData;
 import static google.registry.flows.domain.DomainTransferUtils.createTransferResponse;
 import static google.registry.flows.domain.DomainTransferUtils.createTransferServerApproveEntities;
-import static google.registry.flows.domain.token.AllocationTokenFlowUtils.maybeApplyPackageRemovalToken;
-import static google.registry.flows.domain.token.AllocationTokenFlowUtils.verifyTokenAllowedOnDomain;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_TRANSFER_REQUEST;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -135,8 +133,6 @@ import org.joda.time.DateTime;
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForTldException}
  * @error {@link
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.AlreadyRedeemedAllocationTokenException}
- * @error {@link
- *     google.registry.flows.domain.token.AllocationTokenFlowUtils.MissingRemovePackageTokenOnPackageDomainException}
  */
 @ReportingSpec(ActivityReportField.DOMAIN_TRANSFER_REQUEST)
 public final class DomainTransferRequestFlow implements TransactionalFlow {
@@ -189,9 +185,6 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
             : ((Transfer) resourceCommand).getPeriod();
     verifyTransferAllowed(existingDomain, period, now, superuserExtension, allocationToken);
 
-    // If client passed an applicable static token this updates the domain
-    existingDomain = maybeApplyPackageRemovalToken(existingDomain, allocationToken);
-
     String tld = existingDomain.getTld();
     Registry registry = Registry.get(tld);
     // An optional extension from the client specifying what they think the transfer should cost.
@@ -307,7 +300,6 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
       Optional<AllocationToken> allocationToken)
       throws EppException {
     verifyNoDisallowedStatuses(existingDomain, DISALLOWED_STATUSES);
-    verifyTokenAllowedOnDomain(existingDomain, allocationToken);
     if (!isSuperuser) {
       verifyAuthInfoPresentForResourceTransfer(authInfo);
       verifyAuthInfo(authInfo.get(), existingDomain);

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -182,8 +182,7 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     DomainHistory domainHistory =
         historyBuilder.setType(DOMAIN_UPDATE).setDomain(newDomain).build();
     validateNewState(newDomain);
-    if (!Objects.equals(newDomain.getDsData(), existingDomain.getDsData())
-        || !Objects.equals(newDomain.getNsHosts(), existingDomain.getNsHosts())) {
+    if (requiresDnsUpdate(existingDomain, newDomain)) {
       dnsQueue.addDomainRefreshTask(targetId);
     }
     ImmutableSet.Builder<ImmutableObject> entitiesToSave = new ImmutableSet.Builder<>();
@@ -207,6 +206,16 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     return responseBuilder.build();
   }
 
+  /** Determines if any of the changes to new domain should trigger DNS update. */
+  private boolean requiresDnsUpdate(Domain existingDomain, Domain newDomain) {
+    if (existingDomain.shouldPublishToDns() != newDomain.shouldPublishToDns()
+        || !Objects.equals(newDomain.getDsData(), existingDomain.getDsData())
+        || !Objects.equals(newDomain.getNsHosts(), existingDomain.getNsHosts())) {
+      return true;
+    }
+    return false;
+  }
+
   /** Fail if the object doesn't exist or was deleted. */
   private void verifyUpdateAllowed(Update command, Domain existingDomain, DateTime now)
       throws EppException {

core/src/main/java/google/registry/model/IdService.java

@@ -17,58 +17,114 @@ package google.registry.model;
 import static com.google.common.base.Preconditions.checkState;
 
 import com.google.appengine.api.datastore.DatastoreServiceFactory;
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.flogger.FluentLogger;
 import google.registry.beam.common.RegistryPipelineWorkerInitializer;
 import google.registry.config.RegistryEnvironment;
 import google.registry.model.annotations.DeleteAfterMigration;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
 
 /**
- * Allocates a globally unique {@link Long} number to use as an Ofy {@code @Id}.
+ * Allocates a {@link long} to use as a {@code @Id}, (part) of the primary SQL key for an entity.
  *
- * <p>In non-test, non-beam environments the Id is generated by Datastore, otherwise it's from an
- * atomic long number that's incremented every time this method is called.
+ * <p>Normally, the ID is globally unique and allocated by Datastore. It is possible to override
+ * this behavior by providing an ID supplier, such as in unit tests, where a self-allocated ID based
+ * on a monotonically increasing atomic {@link long} is used. Such an ID supplier can also be used
+ * in other scenarios, such as in a Beam pipeline to get around the limitation of Beam's inability
+ * to use GAE SDK to access Datastore. The override should be used with great care lest it results
+ * in irreversible data corruption.
+ *
+ * @see #setIdSupplier(Supplier)
  */
 @DeleteAfterMigration
 public final class IdService {
 
-  /**
-   * A placeholder String passed into DatastoreService.allocateIds that ensures that all ids are
-   * initialized from the same id pool.
-   */
-  private static final String APP_WIDE_ALLOCATION_KIND = "common";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private IdService() {}
+
+  private static Supplier<Long> idSupplier =
+      RegistryEnvironment.UNITTEST.equals(RegistryEnvironment.get())
+          ? SelfAllocatedIdSupplier.getInstance()
+          : DatastoreIdSupplier.getInstance();
 
   /**
-   * Counts of used ids for use in unit tests or Beam.
+   * Provides a {@link Supplier} of ID that overrides the default.
    *
-   * <p>Note that one should only use self-allocate Ids in Beam for entities whose Ids are not
-   * important and are not persisted back to the database, i. e. nowhere the uniqueness of the ID is
-   * required.
+   * <p>Currently, the only use case for an override is in the Beam pipeline, where access to
+   * Datastore is not possible through the App Engine API. As such, the setter explicitly checks if
+   * the runtime is Beam.
+   *
+   * <p>Because the provided supplier is not guaranteed to be globally unique and compatible with
+   * existing IDs in the database, one should proceed with great care. It is safe to use an
+   * arbitrary supplier when the resulting IDs are not significant and not persisted back to the
+   * database, i.e. the IDs are only required by the {@link Buildable} contract but are not used in
+   * any meaningful way. One example is the RDE pipeline where we project EPP resource entities from
+   * history entries to watermark time, which are then marshalled into XML elements in the RDE
+   * deposits, where the IDs are omitted.
    */
-  private static final AtomicLong nextSelfAllocatedId = new AtomicLong(1); // ids cannot be zero
-
-  private static final boolean isSelfAllocated() {
-    return RegistryEnvironment.UNITTEST.equals(RegistryEnvironment.get())
-        || "true".equals(System.getProperty(RegistryPipelineWorkerInitializer.PROPERTY, "false"));
+  public static void setIdSupplier(Supplier<Long> idSupplier) {
+    checkState(
+        "true".equals(System.getProperty(RegistryPipelineWorkerInitializer.PROPERTY, "false")),
+        "Can only set ID supplier in a Beam pipeline");
+    logger.atWarning().log("Using ID supplier override!");
+    IdService.idSupplier = idSupplier;
   }
 
   /** Allocates an id. */
-  // TODO(b/201547855): Find a way to allocate a unique ID without datastore.
   public static long allocateId() {
-    return isSelfAllocated()
-        ? nextSelfAllocatedId.getAndIncrement()
-        : DatastoreServiceFactory.getDatastoreService()
-            .allocateIds(APP_WIDE_ALLOCATION_KIND, 1)
-            .iterator()
-            .next()
-            .getId();
+    return idSupplier.get();
   }
 
-  /** Resets the global self-allocated id counter (i.e. sets the next id to 1). */
-  @VisibleForTesting
-  public static void resetSelfAllocatedId() {
-    checkState(
-        isSelfAllocated(), "Can only call resetSelfAllocatedId() in unit tests or Beam pipelines");
-    nextSelfAllocatedId.set(1); // ids cannot be zero
+  // TODO(b/201547855): Find a way to allocate a unique ID without datastore.
+  private static class DatastoreIdSupplier implements Supplier<Long> {
+
+    private static final DatastoreIdSupplier INSTANCE = new DatastoreIdSupplier();
+
+    /**
+     * A placeholder String passed into {@code DatastoreService.allocateIds} that ensures that all
+     * IDs are initialized from the same ID pool.
+     */
+    private static final String APP_WIDE_ALLOCATION_KIND = "common";
+
+    public static DatastoreIdSupplier getInstance() {
+      return INSTANCE;
+    }
+
+    @Override
+    public Long get() {
+      return DatastoreServiceFactory.getDatastoreService()
+          .allocateIds(APP_WIDE_ALLOCATION_KIND, 1)
+          .iterator()
+          .next()
+          .getId();
+    }
+  }
+
+  /**
+   * An ID supplier that allocates an ID from a monotonically increasing atomic {@link long}.
+   *
+   * <p>The generated IDs are only unique within the same JVM. It is not suitable for production use
+   * unless in cases the IDs are not significant.
+   */
+  public static class SelfAllocatedIdSupplier implements Supplier<Long> {
+
+    private static final SelfAllocatedIdSupplier INSTANCE = new SelfAllocatedIdSupplier();
+
+    /** Counts of used ids for self allocating IDs. */
+    private static final AtomicLong nextSelfAllocatedId = new AtomicLong(1); // ids cannot be zero
+
+    public static SelfAllocatedIdSupplier getInstance() {
+      return INSTANCE;
+    }
+
+    @Override
+    public Long get() {
+      return nextSelfAllocatedId.getAndIncrement();
+    }
+
+    public void reset() {
+      nextSelfAllocatedId.set(1);
+    }
   }
 }

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -38,7 +38,7 @@ import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.persistence.VKey;
-import google.registry.persistence.WithLongVKey;
+import google.registry.persistence.WithVKey;
 import google.registry.persistence.converter.JodaMoneyType;
 import java.util.Optional;
 import java.util.Set;
@@ -295,7 +295,7 @@ public abstract class BillingEvent extends ImmutableObject
         @Index(columnList = "cancellation_matching_billing_recurrence_id")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
-  @WithLongVKey(compositeKey = true)
+  @WithVKey(Long.class)
   public static class OneTime extends BillingEvent {
 
     /** The billable value. */
@@ -473,7 +473,7 @@ public abstract class BillingEvent extends ImmutableObject
         @Index(columnList = "recurrence_time_of_year")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
-  @WithLongVKey(compositeKey = true)
+  @WithVKey(Long.class)
   public static class Recurring extends BillingEvent {
 
     /**
@@ -606,7 +606,7 @@ public abstract class BillingEvent extends ImmutableObject
         @Index(columnList = "billing_recurrence_id")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
-  @WithLongVKey(compositeKey = true)
+  @WithVKey(Long.class)
   public static class Cancellation extends BillingEvent {
 
     /** The billing time of the charge that is being cancelled. */

core/src/main/java/google/registry/model/bulkquery/DomainLite.java

@@ -17,7 +17,7 @@ package google.registry.model.bulkquery;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainBase;
 import google.registry.persistence.VKey;
-import google.registry.persistence.WithStringVKey;
+import google.registry.persistence.WithVKey;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Entity;
@@ -31,7 +31,7 @@ import javax.persistence.Entity;
  * <p>Please refer to {@link BulkQueryEntities} for more information.
  */
 @Entity(name = "Domain")
-@WithStringVKey
+@WithVKey(String.class)
 @Access(AccessType.FIELD)
 public class DomainLite extends DomainBase {
 

core/src/main/java/google/registry/model/contact/Contact.java

@@ -18,7 +18,7 @@ import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.persistence.VKey;
-import google.registry.persistence.WithStringVKey;
+import google.registry.persistence.WithVKey;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Entity;
@@ -45,7 +45,7 @@ import org.joda.time.DateTime;
       @Index(columnList = "searchName")
     })
 @ExternalMessagingName("contact")
-@WithStringVKey(compositeKey = true)
+@WithVKey(String.class)
 @Access(AccessType.FIELD)
 public class Contact extends ContactBase implements ForeignKeyedEppResource {
 

core/src/main/java/google/registry/model/domain/Domain.java

@@ -22,7 +22,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.domain.secdns.DomainDsData;
 import google.registry.model.host.Host;
 import google.registry.persistence.VKey;
-import google.registry.persistence.WithStringVKey;
+import google.registry.persistence.WithVKey;
 import java.util.Set;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -66,7 +66,7 @@ import org.joda.time.DateTime;
       @Index(columnList = "transfer_billing_event_id"),
       @Index(columnList = "transfer_billing_recurrence_id")
     })
-@WithStringVKey(compositeKey = true)
+@WithVKey(String.class)
 @ExternalMessagingName("domain")
 @Access(AccessType.FIELD)
 public class Domain extends DomainBase implements ForeignKeyedEppResource {

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -41,7 +41,7 @@ import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.DomainHistoryVKey;
 import google.registry.persistence.VKey;
-import google.registry.persistence.WithStringVKey;
+import google.registry.persistence.WithVKey;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
@@ -58,7 +58,7 @@ import org.joda.time.DateTime;
 
 /** An entity representing an allocation token. */
 @Entity
-@WithStringVKey(compositeKey = true)
+@WithVKey(String.class)
 @Table(
     indexes = {
       @Index(columnList = "token", name = "allocation_token_token_idx", unique = true),
@@ -300,6 +300,9 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
           !getInstance().tokenType.equals(TokenType.PACKAGE)
               || getInstance().renewalPriceBehavior.equals(RenewalPriceBehavior.SPECIFIED),
           "Package tokens must have renewalPriceBehavior set to SPECIFIED");
+      checkArgument(
+          !getInstance().tokenType.equals(TokenType.PACKAGE) || !getInstance().discountPremiums,
+          "Package tokens cannot discount premium names");
       checkArgument(
           getInstance().domainName == null || TokenType.SINGLE_USE.equals(getInstance().tokenType),
           "Domain name can only be specified for SINGLE_USE tokens");

core/src/main/java/google/registry/model/host/Host.java

@@ -19,7 +19,7 @@ import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.persistence.VKey;
-import google.registry.persistence.WithStringVKey;
+import google.registry.persistence.WithVKey;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 
@@ -51,7 +51,7 @@ import javax.persistence.AccessType;
       @javax.persistence.Index(columnList = "currentSponsorRegistrarId")
     })
 @ExternalMessagingName("host")
-@WithStringVKey(compositeKey = true)
+@WithVKey(String.class)
 @Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property)
 public class Host extends HostBase implements ForeignKeyedEppResource {
 

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -46,7 +46,7 @@ import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferResponse.ContactTransferResponse;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.persistence.VKey;
-import google.registry.persistence.WithLongVKey;
+import google.registry.persistence.WithVKey;
 import google.registry.util.NullIgnoringCollectionBuilder;
 import java.util.Optional;
 import javax.persistence.AttributeOverride;
@@ -342,7 +342,7 @@ public abstract class PollMessage extends ImmutableObject
    */
   @Entity
   @DiscriminatorValue("ONE_TIME")
-  @WithLongVKey(compositeKey = true)
+  @WithVKey(Long.class)
   public static class OneTime extends PollMessage {
 
     @Embedded
@@ -544,7 +544,7 @@ public abstract class PollMessage extends ImmutableObject
    */
   @Entity
   @DiscriminatorValue("AUTORENEW")
-  @WithLongVKey(compositeKey = true)
+  @WithVKey(Long.class)
   public static class Autorenew extends PollMessage {
 
     /** The target id of the autorenew event. */

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -278,7 +278,7 @@ public abstract class PersistenceModule {
     replicaInstanceConnectionName.ifPresent(
         name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
     overrides.put(
-        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
+        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ.name());
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -294,7 +294,7 @@ public abstract class PersistenceModule {
     replicaInstanceConnectionName.ifPresent(
         name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
     overrides.put(
-        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
+        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ.name());
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 

core/src/main/java/google/registry/persistence/WithStringVKey.java

@@ -1,42 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.persistence;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Target;
-import javax.persistence.AttributeConverter;
-import javax.persistence.Entity;
-
-/**
- * Annotation for {@link Entity} which id is string type and needs an {@link AttributeConverter} for
- * its VKey.
- */
-@Target({ElementType.TYPE})
-public @interface WithStringVKey {
-  /**
-   * Sets the suffix of the class name for the {@link AttributeConverter} generated by
-   * StringVKeyProcessor. If not set, the suffix will be the type name of the VKey. Note that the
-   * class name will be "VKeyConverter_" concatenated with the suffix.
-   */
-  String classNameSuffix() default "";
-
-  /**
-   * Set to true if this is a composite vkey.
-   *
-   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
-   * enclosing class has to take care of that.
-   */
-  boolean compositeKey() default false;
-}

core/src/main/java/google/registry/persistence/WithVKey.java

@@ -1,4 +1,4 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,29 +14,22 @@
 
 package google.registry.persistence;
 
+import java.io.Serializable;
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Target;
 import javax.persistence.AttributeConverter;
 import javax.persistence.Entity;
 
 /**
- * Annotation for {@link Entity} which id is long type and needs an {@link AttributeConverter} for
- * its VKey.
+ * Annotation for {@link Entity} that can be saved as a foreign key in the form of a {@link VKey} in
+ * another table.
+ *
+ * <p>A {@link AttributeConverter} named {@code VKeyConverter_[EntityClassSimpleName]} will be
+ * automatically generated by {@code google.registry.processors.VKeyProcessor}, this class must be
+ * manually added to {@code persistence.xml} in order for it to be picked up by Hibernate.
  */
 @Target({ElementType.TYPE})
-public @interface WithLongVKey {
-  /**
-   * Sets the suffix of the class name for the {@link AttributeConverter} generated by
-   * LongVKeyProcessor. If not set, the suffix will be the type name of the VKey. Note that the
-   * class name will be "VKeyConverter_" concatenated with the suffix.
-   */
-  String classNameSuffix() default "";
-
-  /**
-   * Set to true if this is a composite vkey.
-   *
-   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
-   * enclosing class has to take care of that.
-   */
-  boolean compositeKey() default false;
+public @interface WithVKey {
+  /** The type of the SQL primary ID of the entity that is saved in the {@link VKey} */
+  Class<? extends Serializable> value();
 }

core/src/main/java/google/registry/persistence/converter/VKeyConverter.java

@@ -14,20 +14,34 @@
 
 package google.registry.persistence.converter;
 
-import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import java.io.Serializable;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeConverter;
 
-/** Converts VKey to a string or long column. */
+/**
+ * Converts {@link VKey} to/from a type that can be directly stored in the database.
+ *
+ * <p>Typically the converted type is {@link String} or {@link Long}.
+ */
 public abstract class VKeyConverter<T, C extends Serializable>
     implements AttributeConverter<VKey<? extends T>, C> {
+
   @Override
   @Nullable
-  @SuppressWarnings("unchecked")
   public C convertToDatabaseColumn(@Nullable VKey<? extends T> attribute) {
-    return attribute == null ? null : (C) attribute.getSqlKey();
+    if (attribute == null) {
+      return null;
+    }
+    try {
+      return getKeyClass().cast(attribute.getSqlKey());
+    } catch (ClassCastException e) {
+      throw new RuntimeException(
+          String.format(
+              "Cannot cast SQL key %s of type %s to type %s",
+              attribute.getSqlKey(), attribute.getSqlKey().getClass(), getKeyClass()),
+          e);
+    }
   }
 
   @Override
@@ -36,27 +50,12 @@ public abstract class VKeyConverter<T, C extends Serializable>
     if (dbData == null) {
       return null;
     }
-    Class<T> clazz = getAttributeClass();
-    Key<T> ofyKey;
-    if (!hasCompositeOfyKey()) {
-      // If this isn't a composite key, we can create the Ofy key from the SQL key.
-      ofyKey =
-          dbData instanceof String
-              ? Key.create(clazz, (String) dbData)
-              : Key.create(clazz, (Long) dbData);
-      return VKey.create(clazz, dbData, ofyKey);
-    } else {
-      // We don't know how to create the Ofy key and probably don't have everything necessary to do
-      // it anyway, so just create an asymmetric key - the containing object will have to convert it
-      // into a symmetric key.
-      return VKey.createSql(clazz, dbData);
-    }
+    return VKey.createSql(getEntityClass(), dbData);
   }
 
-  protected boolean hasCompositeOfyKey() {
-    return false;
-  }
+  /** Returns the class of the entity that the VKey represents. */
+  protected abstract Class<T> getEntityClass();
 
-  /** Returns the class of the attribute. */
-  protected abstract Class<T> getAttributeClass();
+  /** Returns the class of the key that the VKey holds. */
+  protected abstract Class<C> getKeyClass();
 }

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerModule.java

@@ -19,7 +19,7 @@ import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
 import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.CredentialModule.ApplicationDefaultCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
@@ -33,7 +33,7 @@ public abstract class SecretManagerModule {
   @Provides
   @Singleton
   static SecretManagerServiceSettings provideSecretManagerSetting(
-      @DefaultCredential GoogleCredentialsBundle credentialsBundle) {
+      @ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle) {
     try {
       return SecretManagerServiceSettings.newBuilder()
           .setCredentialsProvider(() -> credentialsBundle.getGoogleCredentials())

core/src/main/java/google/registry/tools/AppEngineConnection.java

@@ -48,7 +48,7 @@ import org.json.simple.JSONValue;
  * <p>By default - connects to the TOOLS service. To create a Connection to another service, call
  * the {@link #withService} function.
  */
-class AppEngineConnection {
+public class AppEngineConnection {
 
   /** Pattern to heuristically extract title tag contents in HTML responses. */
   private static final Pattern HTML_TITLE_TAG_PATTERN = Pattern.compile("<title>(.*?)</title>");

core/src/main/java/google/registry/tools/AuthModule.java

@@ -35,6 +35,7 @@ import dagger.Binds;
 import dagger.Lazy;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.config.CredentialModule.ApplicationDefaultCredential;
 import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.CredentialModule.LocalCredential;
 import google.registry.config.CredentialModule.LocalCredentialJson;
@@ -228,6 +229,11 @@ public class AuthModule {
     @DefaultCredential
     abstract GoogleCredentialsBundle provideLocalCredentialAsDefaultCredential(
         @LocalCredential GoogleCredentialsBundle credential);
+
+    @Binds
+    @ApplicationDefaultCredential
+    abstract GoogleCredentialsBundle provideLocalCredentialAsApplicationDefaultCredential(
+        @LocalCredential GoogleCredentialsBundle credential);
   }
 
   /** Raised when we need a user login. */

core/src/main/java/google/registry/tools/CommandWithConnection.java

@@ -15,6 +15,6 @@
 package google.registry.tools;
 
 /** A command that can send HTTP requests to a backend module. */
-interface CommandWithConnection extends Command {
+public interface CommandWithConnection extends Command {
   void setConnection(AppEngineConnection connection);
 }

core/src/main/java/google/registry/tools/GetEppResourceCommand.java

@@ -15,30 +15,31 @@
 package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import google.registry.model.EppResource;
+import google.registry.util.Clock;
 import java.util.Optional;
+import javax.inject.Inject;
 import org.joda.time.DateTime;
 
 /** Abstract command to print one or more resources to stdout. */
 @Parameters(separators = " =")
 abstract class GetEppResourceCommand implements CommandWithRemoteApi {
 
-  private final DateTime now = DateTime.now(UTC);
-
   @Parameter(
       names = "--read_timestamp",
       description = "Timestamp to use when reading. May not be in the past.")
-  protected DateTime readTimestamp = now;
+  protected DateTime readTimestamp;
 
   @Parameter(
       names = "--expand",
       description = "Fully expand the requested resource. NOTE: Output may be lengthy.")
   boolean expand;
 
+  @Inject Clock clock;
+
   /** Runs the command's own logic that calls {@link #printResource}. */
   abstract void runAndPrint();
 
@@ -59,7 +60,11 @@ abstract class GetEppResourceCommand implements CommandWithRemoteApi {
 
   @Override
   public void run() {
-    checkArgument(!readTimestamp.isBefore(now), "--read_timestamp may not be in the past");
+    if (readTimestamp == null) {
+      readTimestamp = clock.nowUtc();
+    }
+    checkArgument(
+        !readTimestamp.isBefore(clock.nowUtc()), "--read_timestamp may not be in the past");
     runAndPrint();
   }
 }

core/src/main/java/google/registry/tools/GetPackagePromotionCommand.java

@@ -0,0 +1,47 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import google.registry.model.domain.token.PackagePromotion;
+import java.util.List;
+
+/** Command to show a {@link PackagePromotion} object. */
+@Parameters(separators = " =", commandDescription = "Show package promotion object(s)")
+public class GetPackagePromotionCommand extends GetEppResourceCommand {
+
+  @Parameter(description = "Package token(s)", required = true)
+  private List<String> mainParameters;
+
+  @Override
+  void runAndPrint() {
+    for (String token : mainParameters) {
+      jpaTm()
+          .transact(
+              () -> {
+                PackagePromotion packagePromotion =
+                    checkArgumentPresent(
+                        PackagePromotion.loadByTokenString(token),
+                        "PackagePromotion with package token %s does not exist",
+                        token);
+                System.out.println(packagePromotion);
+              });
+    }
+  }
+}

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -17,6 +17,7 @@ package google.registry.tools;
 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
 import google.registry.tools.javascrap.CreateCancellationsForOneTimesCommand;
+import google.registry.tools.javascrap.CreateSyntheticDomainHistoriesCommand;
 
 /** Container class to create and run remote commands against a Datastore instance. */
 public final class RegistryTool {
@@ -47,6 +48,7 @@ public final class RegistryTool {
           .put("create_registrar", CreateRegistrarCommand.class)
           .put("create_registrar_groups", CreateRegistrarGroupsCommand.class)
           .put("create_reserved_list", CreateReservedListCommand.class)
+          .put("create_synthetic_domain_histories", CreateSyntheticDomainHistoriesCommand.class)
           .put("create_tld", CreateTldCommand.class)
           .put("curl", CurlCommand.class)
           .put("delete_allocation_tokens", DeleteAllocationTokensCommand.class)
@@ -71,6 +73,7 @@ public final class RegistryTool {
           .put("get_history_entries", GetHistoryEntriesCommand.class)
           .put("get_host", GetHostCommand.class)
           .put("get_keyring_secret", GetKeyringSecretCommand.class)
+          .put("get_package_promotion", GetPackagePromotionCommand.class)
           .put("get_premium_list", GetPremiumListCommand.class)
           .put("get_registrar", GetRegistrarCommand.class)
           .put("get_reserved_list", GetReservedListCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -43,6 +43,7 @@ import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
 import google.registry.tools.javascrap.CreateCancellationsForOneTimesCommand;
+import google.registry.tools.javascrap.CreateSyntheticDomainHistoriesCommand;
 import google.registry.util.UtilsModule;
 import google.registry.whois.NonCachingWhoisModule;
 import javax.annotation.Nullable;
@@ -106,6 +107,8 @@ interface RegistryToolComponent {
 
   void inject(CreateRegistrarCommand command);
 
+  void inject(CreateSyntheticDomainHistoriesCommand command);
+
   void inject(CreateTldCommand command);
 
   void inject(EncryptEscrowDepositCommand command);
@@ -118,6 +121,14 @@ interface RegistryToolComponent {
 
   void inject(GenerateEscrowDepositCommand command);
 
+  void inject(GetContactCommand command);
+
+  void inject(GetDomainCommand command);
+
+  void inject(GetHostCommand command);
+
+  void inject(GetPackagePromotionCommand command);
+
   void inject(GetKeyringSecretCommand command);
 
   void inject(GetSqlCredentialCommand command);
@@ -144,6 +155,8 @@ interface RegistryToolComponent {
 
   void inject(SetupOteCommand command);
 
+  void inject(UniformRapidSuspensionCommand command);
+
   void inject(UnlockDomainCommand command);
 
   void inject(UnrenewDomainCommand command);

core/src/main/java/google/registry/tools/RemoteApiOptionsUtil.java

@@ -28,8 +28,8 @@ import java.lang.reflect.Method;
  * {@link RemoteApiOptions} with a JSON representing a user credential.
  */
 public class RemoteApiOptionsUtil {
-  static RemoteApiOptions useGoogleCredentialStream(RemoteApiOptions options, InputStream stream)
-      throws Exception {
+  public static RemoteApiOptions useGoogleCredentialStream(
+      RemoteApiOptions options, InputStream stream) throws Exception {
     Method method =
         options.getClass().getDeclaredMethod("useGoogleCredentialStream", InputStream.class);
     checkState(

core/src/main/java/google/registry/tools/UniformRapidSuspensionCommand.java

@@ -21,7 +21,6 @@ import static google.registry.model.EppResourceUtils.checkResourcesExist;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
-import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
@@ -38,11 +37,13 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.Host;
 import google.registry.tools.soy.DomainRenewSoyInfo;
 import google.registry.tools.soy.UniformRapidSuspensionSoyInfo;
+import google.registry.util.Clock;
 import google.registry.util.DomainNameUtils;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import javax.inject.Inject;
 import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
 import org.joda.time.DateTime;
 import org.joda.time.format.DateTimeFormat;
@@ -119,10 +120,12 @@ final class UniformRapidSuspensionCommand extends MutatingEppToolCommand {
   /** Set of status values to remove. */
   ImmutableSet<String> removeStatuses;
 
+  @Inject Clock clock;
+
   @Override
   protected void initMutatingEppToolCommand() {
     superuser = true;
-    DateTime now = DateTime.now(UTC);
+    DateTime now = clock.nowUtc();
     ImmutableList<String> newCanonicalHosts =
         newHosts.stream().map(DomainNameUtils::canonicalizeHostname).collect(toImmutableList());
     ImmutableSet<String> newHostsSet = ImmutableSet.copyOf(newCanonicalHosts);

core/src/main/java/google/registry/tools/javascrap/CreateSyntheticDomainHistoriesCommand.java

@@ -0,0 +1,209 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.beust.jcommander.Parameters;
+import com.google.appengine.tools.remoteapi.RemoteApiInstaller;
+import com.google.appengine.tools.remoteapi.RemoteApiOptions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.domain.Domain;
+import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import google.registry.tools.AppEngineConnection;
+import google.registry.tools.CommandWithConnection;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.tools.RemoteApiOptionsUtil;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+
+/**
+ * Command that creates an additional synthetic history object for domains.
+ *
+ * <p>This is created to fix the issue identified in b/248112997. After b/245940594, there were some
+ * domains where the most recent history object did not represent the state of the domain as it
+ * exists in the world. Because RDE loads only from DomainHistory objects, this means that RDE was
+ * producing wrong data. This command mitigates that issue by creating synthetic history events for
+ * every domain that was not deleted as of the start of the bad {@link
+ * google.registry.beam.resave.ResaveAllEppResourcesPipeline} -- then, we can guarantee that this
+ * new history object represents the state of the domain as far as we know.
+ *
+ * <p>A previous run of this command (in pipeline form) attempted to do this and succeeded in most
+ * cases. Unfortunately, that pipeline had an issue where it used self-allocated IDs for some of the
+ * dependent objects (e.g. {@link google.registry.model.domain.secdns.DomainDsDataHistory}). As a
+ * result, we want to run this again as a command using Datastore-allocated IDs to re-create
+ * synthetic history objects for any domain whose last history object is one of the
+ * potentially-incorrect synthetic objects.
+ *
+ * <p>We further restrict the domains to domains whose latest history object is before October 4.
+ * This is an arbitrary date that is suitably far after the previous incorrect run of this synthetic
+ * history pipeline, with the purpose of making future runs of this command idempotent (in case the
+ * command fails, we can just run it again and again).
+ */
+@Parameters(
+    separators = " =",
+    commandDescription = "Create synthetic domain history objects to fix RDE.")
+public class CreateSyntheticDomainHistoriesCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi, CommandWithConnection {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static final String HISTORY_REASON =
+      "Create synthetic domain histories to fix RDE for b/248112997";
+  private static final DateTime BAD_PIPELINE_END_TIME = DateTime.parse("2022-09-10T12:00:00.000Z");
+  private static final DateTime NEW_SYNTHETIC_ROUND_START =
+      DateTime.parse("2022-10-04T00:00:00.000Z");
+
+  private static final ExecutorService executor = Executors.newFixedThreadPool(20);
+  private static final AtomicInteger numDomainsProcessed = new AtomicInteger();
+
+  private AppEngineConnection connection;
+
+  @Inject
+  @Config("registryAdminClientId")
+  String registryAdminRegistrarId;
+
+  @Inject @CredentialModule.LocalCredentialJson String localCredentialJson;
+
+  private final ThreadLocal<RemoteApiInstaller> installerThreadLocal =
+      ThreadLocal.withInitial(this::createInstaller);
+
+  private ImmutableSet<String> domainRepoIds;
+
+  @Override
+  protected String prompt() {
+    jpaTm()
+        .transact(
+            () -> {
+              domainRepoIds =
+                  jpaTm()
+                      .query(
+                          "SELECT dh.domainRepoId FROM DomainHistory dh JOIN Tld t ON t.tldStr ="
+                              + " dh.domainBase.tld WHERE t.tldType = 'REAL' AND dh.type ="
+                              + " 'SYNTHETIC' AND dh.modificationTime > :badPipelineEndTime AND"
+                              + " dh.modificationTime < :newSyntheticRoundStart AND"
+                              + " (dh.domainRepoId, dh.modificationTime) IN (SELECT domainRepoId,"
+                              + " MAX(modificationTime) FROM DomainHistory GROUP BY domainRepoId)",
+                          String.class)
+                      .setParameter("badPipelineEndTime", BAD_PIPELINE_END_TIME)
+                      .setParameter("newSyntheticRoundStart", NEW_SYNTHETIC_ROUND_START)
+                      .getResultStream()
+                      .collect(toImmutableSet());
+            });
+    return String.format(
+        "Attempt to create synthetic history entries for %d domains?", domainRepoIds.size());
+  }
+
+  @Override
+  protected String execute() throws Exception {
+    List<Future<?>> futures = new ArrayList<>();
+    for (String domainRepoId : domainRepoIds) {
+      futures.add(
+          executor.submit(
+              () -> {
+                // Make sure the remote API is installed for ID generation
+                installerThreadLocal.get();
+                jpaTm()
+                    .transact(
+                        () -> {
+                          Domain domain =
+                              jpaTm().loadByKey(VKey.createSql(Domain.class, domainRepoId));
+                          jpaTm()
+                              .put(
+                                  HistoryEntry.createBuilderForResource(domain)
+                                      .setRegistrarId(registryAdminRegistrarId)
+                                      .setBySuperuser(true)
+                                      .setRequestedByRegistrar(false)
+                                      .setModificationTime(jpaTm().getTransactionTime())
+                                      .setReason(HISTORY_REASON)
+                                      .setType(HistoryEntry.Type.SYNTHETIC)
+                                      .build());
+                        });
+                int numProcessed = numDomainsProcessed.incrementAndGet();
+                if (numProcessed % 1000 == 0) {
+                  System.out.printf("Saved histories for %d domains%n", numProcessed);
+                }
+                return null;
+              }));
+    }
+    for (Future<?> future : futures) {
+      try {
+        future.get();
+      } catch (Exception e) {
+        logger.atSevere().withCause(e).log("Error");
+      }
+    }
+    executor.shutdown();
+    executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+    return String.format("Saved entries for %d domains", numDomainsProcessed.get());
+  }
+
+  @Override
+  public void setConnection(AppEngineConnection connection) {
+    this.connection = connection;
+  }
+
+  /**
+   * Installs the remote API so that the worker threads can use Datastore for ID generation.
+   *
+   * <p>Lifted from the RegistryCli class
+   */
+  private RemoteApiInstaller createInstaller() {
+    RemoteApiInstaller installer = new RemoteApiInstaller();
+    RemoteApiOptions options = new RemoteApiOptions();
+    options.server(connection.getServer().getHost(), getPort(connection.getServer()));
+    if (RegistryConfig.areServersLocal()) {
+      // Use dev credentials for localhost.
+      options.useDevelopmentServerCredential();
+    } else {
+      try {
+        RemoteApiOptionsUtil.useGoogleCredentialStream(
+            options, new ByteArrayInputStream(localCredentialJson.getBytes(UTF_8)));
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+    try {
+      installer.install(options);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    ObjectifyService.initOfy();
+    return installer;
+  }
+
+  private static int getPort(URL url) {
+    return url.getPort() == -1 ? url.getDefaultPort() : url.getPort();
+  }
+}

core/src/main/java/google/registry/tools/javascrap/CreateSyntheticDomainHistoriesPipeline.java

@@ -1,130 +0,0 @@
-// Copyright 2022 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-
-import com.google.common.collect.ImmutableMap;
-import dagger.Component;
-import google.registry.beam.common.RegistryJpaIO;
-import google.registry.beam.common.RegistryPipelineOptions;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.config.RegistryConfig.ConfigModule;
-import google.registry.model.domain.Domain;
-import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
-import google.registry.persistence.VKey;
-import java.io.Serializable;
-import javax.inject.Singleton;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.joda.time.DateTime;
-
-/**
- * Pipeline that creates a synthetic history for every non-deleted {@link Domain} in SQL.
- *
- * <p>This is created to fix the issue identified in b/248112997. After b/245940594, there were some
- * domains where the most recent history object did not represent the state of the domain as it
- * exists in the world. Because RDE loads only from DomainHistory objects, this means that RDE was
- * producing wrong data. This pipeline mitigates that issue by creating synthetic history events for
- * every domain that was not deleted as of the start of the pipeline -- then, we can guarantee that
- * this new history object represents the state of the domain as far as we know.
- *
- * <p>To run the pipeline (replace the environment as appropriate):
- *
- * <p><code>
- * $ ./nom_build :core:createSyntheticDomainHistories --args="--region=us-central1
- *   --runner=DataflowRunner
- *   --registryEnvironment=CRASH
- *   --project={project-id}
- *   --workerMachineType=n2-standard-4"
- * </code>
- */
-public class CreateSyntheticDomainHistoriesPipeline implements Serializable {
-
-  private static final String HISTORY_REASON =
-      "Create synthetic domain histories to fix RDE for b/248112997";
-  private static final DateTime BAD_PIPELINE_START_TIME =
-      DateTime.parse("2022-09-05T00:00:00.000Z");
-
-  static void setup(Pipeline pipeline, String registryAdminRegistrarId) {
-    pipeline
-        .apply(
-            "Read all domain repo IDs",
-            RegistryJpaIO.read(
-                "SELECT repoId FROM Domain WHERE deletionTime > :badPipelineStartTime",
-                ImmutableMap.of("badPipelineStartTime", BAD_PIPELINE_START_TIME),
-                String.class,
-                repoId -> VKey.createSql(Domain.class, repoId)))
-        .apply(
-            "Save a synthetic DomainHistory for each domain",
-            ParDo.of(new DomainHistoryCreator(registryAdminRegistrarId)));
-  }
-
-  private static class DomainHistoryCreator extends DoFn<VKey<Domain>, Void> {
-
-    private final String registryAdminRegistrarId;
-
-    private DomainHistoryCreator(String registryAdminRegistrarId) {
-      this.registryAdminRegistrarId = registryAdminRegistrarId;
-    }
-
-    @ProcessElement
-    public void processElement(
-        @Element VKey<Domain> key, PipelineOptions options, OutputReceiver<Void> outputReceiver) {
-      jpaTm()
-          .transact(
-              () -> {
-                Domain domain = jpaTm().loadByKey(key);
-                jpaTm()
-                    .put(
-                        HistoryEntry.createBuilderForResource(domain)
-                            .setRegistrarId(registryAdminRegistrarId)
-                            .setBySuperuser(true)
-                            .setRequestedByRegistrar(false)
-                            .setModificationTime(jpaTm().getTransactionTime())
-                            .setReason(HISTORY_REASON)
-                            .setType(HistoryEntry.Type.SYNTHETIC)
-                            .build());
-                outputReceiver.output(null);
-              });
-    }
-  }
-
-  public static void main(String[] args) {
-    RegistryPipelineOptions options =
-        PipelineOptionsFactory.fromArgs(args).withValidation().as(RegistryPipelineOptions.class);
-    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
-    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
-    String registryAdminRegistrarId =
-        DaggerCreateSyntheticDomainHistoriesPipeline_ConfigComponent.create()
-            .getRegistryAdminRegistrarId();
-
-    Pipeline pipeline = Pipeline.create(options);
-    setup(pipeline, registryAdminRegistrarId);
-    pipeline.run();
-  }
-
-  @Singleton
-  @Component(modules = ConfigModule.class)
-  interface ConfigComponent {
-
-    @Config("registryAdminClientId")
-    String getRegistryAdminRegistrarId();
-  }
-}

core/src/test/java/google/registry/batch/CheckPackagesComplianceActionTest.java

@@ -0,0 +1,243 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package google.registry.batch;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.persistActiveContact;
+import static google.registry.testing.DatabaseHelper.persistEppResource;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.testing.LogsSubject.assertAboutLogs;
+
+import com.google.common.testing.TestLogHandler;
+import google.registry.model.billing.BillingEvent.RenewalPriceBehavior;
+import google.registry.model.contact.Contact;
+import google.registry.model.domain.Domain;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.domain.token.AllocationToken.TokenType;
+import google.registry.model.domain.token.PackagePromotion;
+import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DatabaseHelper;
+import google.registry.testing.FakeClock;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.joda.money.CurrencyUnit;
+import org.joda.money.Money;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableSet;
+
+/** Unit tests for {@link CheckPackagesComplianceAction}. */
+public class CheckPackagesComplianceActionTest {
+  // This is the default creation time for test data.
+  private final FakeClock clock = new FakeClock(DateTime.parse("2012-03-25TZ"));
+
+  @RegisterExtension
+  public final AppEngineExtension appEngine =
+      AppEngineExtension.builder().withCloudSql().withClock(clock).build();
+
+  private CheckPackagesComplianceAction action;
+  private AllocationToken token;
+  private final TestLogHandler logHandler = new TestLogHandler();
+  private final Logger loggerToIntercept =
+      Logger.getLogger(CheckPackagesComplianceAction.class.getCanonicalName());
+
+  private Contact contact;
+  private PackagePromotion packagePromotion;
+
+  @BeforeEach
+  void beforeEach() {
+    loggerToIntercept.addHandler(logHandler);
+    createTld("tld");
+    action = new CheckPackagesComplianceAction();
+    token =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("abc123")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    packagePromotion =
+        new PackagePromotion.Builder()
+            .setToken(token)
+            .setMaxDomains(3)
+            .setMaxCreates(1)
+            .setPackagePrice(Money.of(CurrencyUnit.USD, 1000))
+            .setNextBillingDate(DateTime.parse("2012-11-12T05:00:00Z"))
+            .setLastNotificationSent(DateTime.parse("2010-11-12T05:00:00Z"))
+            .build();
+
+    jpaTm().transact(() -> jpaTm().put(packagePromotion));
+    contact = persistActiveContact("contact1234");
+  }
+
+  @AfterEach
+  void afterEach() {
+    loggerToIntercept.removeHandler(logHandler);
+  }
+
+  @Test
+  void testSuccess_noPackageOverCreateLimit() {
+    Domain domain1 =
+        persistEppResource(
+            DatabaseHelper.newDomain("foo.tld", contact)
+                .asBuilder()
+                .setCurrentPackageToken(token.createVKey())
+                .build());
+
+    action.run();
+    assertAboutLogs()
+        .that(logHandler)
+        .hasLogAtLevelWithMessage(Level.INFO, "Found no packages over their create limit.");
+  }
+
+  @Test
+  void testSuccess_onePackageOverCreateLimit() {
+    // Create limit is 1, creating 2 domains to go over the limit
+    persistEppResource(
+        DatabaseHelper.newDomain("foo.tld", contact)
+            .asBuilder()
+            .setCurrentPackageToken(token.createVKey())
+            .build());
+    persistEppResource(
+        DatabaseHelper.newDomain("buzz.tld", contact)
+            .asBuilder()
+            .setCurrentPackageToken(token.createVKey())
+            .build());
+
+    action.run();
+    assertAboutLogs()
+        .that(logHandler)
+        .hasLogAtLevelWithMessage(Level.INFO, "Found 1 packages over their create limit.");
+    assertAboutLogs()
+        .that(logHandler)
+        .hasLogAtLevelWithMessage(
+            Level.INFO,
+            "Package with package token abc123 has exceeded their max domain creation limit by 1"
+                + " name(s).");
+  }
+
+  @Test
+  void testSuccess_multiplePackagesOverCreateLimit() {
+    // Create limit is 1, creating 2 domains to go over the limit
+    persistEppResource(
+        DatabaseHelper.newDomain("foo.tld", contact)
+            .asBuilder()
+            .setCurrentPackageToken(token.createVKey())
+            .build());
+    persistEppResource(
+        DatabaseHelper.newDomain("buzz.tld", contact)
+            .asBuilder()
+            .setCurrentPackageToken(token.createVKey())
+            .build());
+
+    AllocationToken token2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("token")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    PackagePromotion packagePromotion2 =
+        new PackagePromotion.Builder()
+            .setToken(token2)
+            .setMaxDomains(8)
+            .setMaxCreates(1)
+            .setPackagePrice(Money.of(CurrencyUnit.USD, 1000))
+            .setNextBillingDate(DateTime.parse("2012-11-12T05:00:00Z"))
+            .build();
+    jpaTm().transact(() -> jpaTm().put(packagePromotion2));
+
+    persistEppResource(
+        DatabaseHelper.newDomain("foo2.tld", contact)
+            .asBuilder()
+            .setCurrentPackageToken(token2.createVKey())
+            .build());
+    persistEppResource(
+        DatabaseHelper.newDomain("buzz2.tld", contact)
+            .asBuilder()
+            .setCurrentPackageToken(token2.createVKey())
+            .build());
+    action.run();
+    assertAboutLogs()
+        .that(logHandler)
+        .hasLogAtLevelWithMessage(Level.INFO, "Found 2 packages over their create limit.");
+
+    assertAboutLogs()
+        .that(logHandler)
+        .hasLogAtLevelWithMessage(
+            Level.INFO,
+            "Package with package token abc123 has exceeded their max domain creation limit by 1"
+                + " name(s).");
+
+    assertAboutLogs()
+        .that(logHandler)
+        .hasLogAtLevelWithMessage(
+            Level.INFO,
+            "Package with package token token has exceeded their max domain creation limit by 1"
+                + " name(s).");
+  }
+
+  @Test
+  void testSuccess_onlyChecksCurrentBillingYear() {
+    AllocationToken token2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("token")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    PackagePromotion packagePromotion2 =
+        new PackagePromotion.Builder()
+            .setToken(token2)
+            .setMaxDomains(8)
+            .setMaxCreates(1)
+            .setPackagePrice(Money.of(CurrencyUnit.USD, 1000))
+            .setNextBillingDate(DateTime.parse("2015-11-12T05:00:00Z"))
+            .build();
+    jpaTm().transact(() -> jpaTm().put(packagePromotion2));
+
+    // Create limit is 1, creating 2 domains to go over the limit
+    persistEppResource(
+        DatabaseHelper.newDomain("foo.tld", contact)
+            .asBuilder()
+            .setCurrentPackageToken(token2.createVKey())
+            .build());
+    persistEppResource(
+        DatabaseHelper.newDomain("buzz.tld", contact)
+            .asBuilder()
+            .setCurrentPackageToken(token2.createVKey())
+            .build());
+
+    action.run();
+    assertAboutLogs()
+        .that(logHandler)
+        .hasLogAtLevelWithMessage(Level.INFO, "Found no packages over their create limit.");
+  }
+}

core/src/test/java/google/registry/flows/domain/DomainTransferRequestFlowTest.java

@@ -21,9 +21,6 @@ import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
-import static google.registry.model.billing.BillingEvent.RenewalPriceBehavior.DEFAULT;
-import static google.registry.model.billing.BillingEvent.RenewalPriceBehavior.SPECIFIED;
-import static google.registry.model.domain.token.AllocationToken.TokenType.PACKAGE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.UNLIMITED_USE;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.TRANSFER_SUCCESSFUL;
@@ -64,7 +61,6 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Streams;
-import com.google.common.truth.Truth8;
 import com.googlecode.objectify.Key;
 import google.registry.batch.ResaveEntityAction;
 import google.registry.flows.EppException;
@@ -89,7 +85,6 @@ import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTok
 import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForTldException;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils.AlreadyRedeemedAllocationTokenException;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils.InvalidAllocationTokenException;
-import google.registry.flows.domain.token.AllocationTokenFlowUtils.MissingRemovePackageTokenOnPackageDomainException;
 import google.registry.flows.exceptions.AlreadyPendingTransferException;
 import google.registry.flows.exceptions.InvalidTransferPeriodValueException;
 import google.registry.flows.exceptions.MissingTransferRequestAuthInfoException;
@@ -1812,84 +1807,4 @@ class DomainTransferRequestFlowTest
         assertThrows(AlreadyRedeemedAllocationTokenException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
-
-  @Test
-  void testFailsPackageDomainInvalidAllocationToken() throws Exception {
-    AllocationToken token =
-        persistResource(
-            new AllocationToken.Builder()
-                .setToken("abc123")
-                .setTokenType(PACKAGE)
-                .setAllowedRegistrarIds(ImmutableSet.of("NewRegistrar"))
-                .setAllowedTlds(ImmutableSet.of("example", "tld"))
-                .setRenewalPriceBehavior(SPECIFIED)
-                .build());
-    setupDomain("example", "tld");
-    persistResource(
-        reloadResourceByForeignKey()
-            .asBuilder()
-            .setCurrentPackageToken(token.createVKey())
-            .build());
-
-    setEppInput("domain_transfer_request_allocation_token.xml", ImmutableMap.of("TOKEN", "abc123"));
-    assertThrows(MissingRemovePackageTokenOnPackageDomainException.class, this::runFlow);
-  }
-
-  @Test
-  void testFailsToTransferPackageDomainNoRemovePackageToken() throws Exception {
-    AllocationToken token =
-        persistResource(
-            new AllocationToken.Builder()
-                .setToken("abc123")
-                .setTokenType(PACKAGE)
-                .setAllowedRegistrarIds(ImmutableSet.of("NewRegistrar"))
-                .setAllowedTlds(ImmutableSet.of("example", "tld"))
-                .setRenewalPriceBehavior(SPECIFIED)
-                .build());
-    setupDomain("example", "tld");
-    persistResource(
-        reloadResourceByForeignKey()
-            .asBuilder()
-            .setCurrentPackageToken(token.createVKey())
-            .build());
-
-    setEppInput("domain_transfer_request.xml");
-    assertThrows(MissingRemovePackageTokenOnPackageDomainException.class, this::runFlow);
-  }
-
-  @Test
-  void testSuccesfullyAppliesRemovePackageToken() throws Exception {
-    setupDomain("example", "tld");
-    AllocationToken token =
-        persistResource(
-            new AllocationToken.Builder()
-                .setToken("abc123")
-                .setTokenType(PACKAGE)
-                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
-                .setAllowedTlds(ImmutableSet.of("tld"))
-                .setRenewalPriceBehavior(SPECIFIED)
-                .build());
-    domain = loadByEntity(domain);
-    persistResource(
-        loadByKey(domain.getAutorenewBillingEvent())
-            .asBuilder()
-            .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
-            .setRenewalPrice(Money.of(USD, new BigDecimal("10.00")))
-            .build());
-    persistResource(
-        reloadResourceByForeignKey()
-            .asBuilder()
-            .setCurrentPackageToken(token.createVKey())
-            .build());
-
-    doSuccessfulTest(
-        "domain_transfer_request_allocation_token.xml",
-        "domain_transfer_request_response.xml",
-        ImmutableMap.of("TOKEN", "__REMOVEPACKAGE__"));
-    Domain domain = reloadResourceByForeignKey();
-    Truth8.assertThat(domain.getCurrentPackageToken()).isEmpty();
-    RenewalPriceBehavior priceBehavior =
-        loadByKey(domain.getAutorenewBillingEvent()).getRenewalPriceBehavior();
-    assertThat(priceBehavior).isEqualTo(DEFAULT);
-  }
 }

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -1747,8 +1747,50 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
   }
 
   @Test
-  void testDnsTaskIsNotTriggeredWhenNoDSChangeSubmitted() {
-    setEppInput("domain_update_no_ds_change.xml");
+  void testAddDnsPublishStatus_enqueueDnsTask() throws Exception {
+    setEppInput(
+        "domain_update_status_change.xml",
+        ImmutableMap.of("STATUS_ADD", "clientHold", "STATUS_REM", "clientTransferProhibited"));
+    persistReferencedEntities();
+    persistResource(
+        persistDomain()
+            .asBuilder()
+            .setDomainName("example.tld")
+            .setStatusValues(ImmutableSet.of(StatusValue.CLIENT_TRANSFER_PROHIBITED))
+            .build());
+    runFlowAsSuperuser();
+    assertDnsTasksEnqueued("example.tld");
+  }
+
+  @Test
+  void testRemoveEveryDnsPublishStatus_enqueueDnsTask() throws Exception {
+    setEppInput(
+        "domain_update_status_change.xml",
+        ImmutableMap.of("STATUS_REM", "serverHold", "STATUS_ADD", "clientTransferProhibited"));
+    persistReferencedEntities();
+    persistResource(
+        persistDomain()
+            .asBuilder()
+            .setDomainName("example.tld")
+            .setStatusValues(ImmutableSet.of(StatusValue.SERVER_HOLD))
+            .build());
+    runFlowAsSuperuser();
+    assertDnsTasksEnqueued("example.tld");
+  }
+
+  @Test
+  void testChangeSomeOrNoChangeDnsPublishStatus_doNotEnqueueDnsTask() throws Exception {
+    setEppInput(
+        "domain_update_status_change.xml",
+        ImmutableMap.of("STATUS_ADD", "clientUpdateProhibited", "STATUS_REM", "pendingDelete"));
+    persistReferencedEntities();
+    persistResource(
+        persistDomain()
+            .asBuilder()
+            .setDomainName("example.tld")
+            .setStatusValues(ImmutableSet.of(StatusValue.PENDING_DELETE, StatusValue.SERVER_HOLD))
+            .build());
+    runFlowAsSuperuser();
     assertNoDnsTasksEnqueued();
   }
 }

core/src/test/java/google/registry/model/domain/token/AllocationTokenTest.java

@@ -228,6 +228,18 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("Package tokens must have renewalPriceBehavior set to SPECIFIED");
   }
 
+  @Test
+  void testFail_packageTokenDiscountPremium() {
+    AllocationToken.Builder builder =
+        new AllocationToken.Builder()
+            .setToken("abc123")
+            .setTokenType(TokenType.PACKAGE)
+            .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+            .setDiscountPremiums(true);
+    IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, builder::build);
+    assertThat(thrown).hasMessageThat().isEqualTo("Package tokens cannot discount premium names");
+  }
+
   @Test
   void testBuild_DomainNameWithLessThanTwoParts() {
     IllegalArgumentException thrown =

core/src/test/java/google/registry/persistence/converter/LongVKeyConverterTest.java

@@ -1,86 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.persistence.converter;
-
-import static com.google.common.truth.Truth.assertThat;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.testing.DatabaseHelper.insertInDb;
-
-import google.registry.model.ImmutableObject;
-import google.registry.persistence.VKey;
-import google.registry.persistence.WithLongVKey;
-import google.registry.testing.AppEngineExtension;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-/** Test SQL persistence of VKey. */
-public class LongVKeyConverterTest {
-
-  @RegisterExtension
-  public final AppEngineExtension appEngineExtension =
-      new AppEngineExtension.Builder()
-          .withCloudSql()
-          .withoutCannedData()
-          .withJpaUnitTestEntities(
-              TestLongEntity.class,
-              VKeyConverter_LongType.class,
-              VKeyConverter_CompositeLongType.class)
-          .withOfyTestEntities(TestLongEntity.class, CompositeKeyTestLongEntity.class)
-          .build();
-
-  @Test
-  void testRoundTrip() {
-    TestLongEntity original =
-        new TestLongEntity(
-            VKey.createSql(TestLongEntity.class, 10L),
-            VKey.createSql(CompositeKeyTestLongEntity.class, 20L));
-    insertInDb(original);
-
-    TestLongEntity retrieved =
-        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestLongEntity.class, "id"));
-    assertThat(retrieved.number.getSqlKey()).isEqualTo(10L);
-    assertThat(retrieved.number.getOfyKey().getId()).isEqualTo(10L);
-
-    assertThat(retrieved.composite.getSqlKey()).isEqualTo(20L);
-    assertThat(retrieved.composite.maybeGetOfyKey().isPresent()).isFalse();
-  }
-
-  @Entity(name = "TestLongEntity")
-  @com.googlecode.objectify.annotation.Entity
-  @WithLongVKey(classNameSuffix = "LongType")
-  static class TestLongEntity extends ImmutableObject {
-    @com.googlecode.objectify.annotation.Id @Id String id = "id";
-
-    VKey<TestLongEntity> number;
-    VKey<CompositeKeyTestLongEntity> composite;
-
-    TestLongEntity(VKey<TestLongEntity> number, VKey<CompositeKeyTestLongEntity> composite) {
-      this.number = number;
-      this.composite = composite;
-    }
-
-    /** Default constructor, needed for hibernate. */
-    public TestLongEntity() {}
-  }
-
-  @Entity(name = "CompositeKeyTestLongEntity")
-  @com.googlecode.objectify.annotation.Entity
-  @WithLongVKey(classNameSuffix = "CompositeLongType", compositeKey = true)
-  static class CompositeKeyTestLongEntity {
-    @com.googlecode.objectify.annotation.Id @Id String id = "id";
-  }
-}

core/src/test/java/google/registry/persistence/converter/StringVKeyConverterTest.java

@@ -1,91 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.persistence.converter;
-
-import static com.google.common.truth.Truth.assertThat;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.testing.DatabaseHelper.insertInDb;
-
-import google.registry.model.ImmutableObject;
-import google.registry.persistence.VKey;
-import google.registry.persistence.WithStringVKey;
-import google.registry.testing.AppEngineExtension;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-/** Test SQL persistence of VKey. */
-public class StringVKeyConverterTest {
-
-  @RegisterExtension
-  public final AppEngineExtension appEngineExtension =
-      new AppEngineExtension.Builder()
-          .withCloudSql()
-          .withoutCannedData()
-          .withJpaUnitTestEntities(
-              TestStringEntity.class,
-              VKeyConverter_StringType.class,
-              VKeyConverter_CompositeStringType.class)
-          .withOfyTestEntities(TestStringEntity.class, CompositeKeyTestStringEntity.class)
-          .build();
-
-  @Test
-  void testRoundTrip() {
-    TestStringEntity original =
-        new TestStringEntity(
-            "TheRealSpartacus",
-            VKey.createSql(TestStringEntity.class, "ImSpartacus!"),
-            VKey.createSql(CompositeKeyTestStringEntity.class, "NoImSpartacus!"));
-    insertInDb(original);
-
-    TestStringEntity retrieved =
-        jpaTm()
-            .transact(
-                () -> jpaTm().getEntityManager().find(TestStringEntity.class, "TheRealSpartacus"));
-    assertThat(retrieved.other.getSqlKey()).isEqualTo("ImSpartacus!");
-    assertThat(retrieved.other.getOfyKey().getName()).isEqualTo("ImSpartacus!");
-
-    assertThat(retrieved.composite.getSqlKey()).isEqualTo("NoImSpartacus!");
-    assertThat(retrieved.composite.maybeGetOfyKey().isPresent()).isFalse();
-  }
-
-  @Entity(name = "TestStringEntity")
-  @com.googlecode.objectify.annotation.Entity
-  @WithStringVKey(classNameSuffix = "StringType")
-  static class TestStringEntity extends ImmutableObject {
-    @com.googlecode.objectify.annotation.Id @Id String id;
-
-    VKey<TestStringEntity> other;
-    VKey<CompositeKeyTestStringEntity> composite;
-
-    TestStringEntity(
-        String id, VKey<TestStringEntity> other, VKey<CompositeKeyTestStringEntity> composite) {
-      this.id = id;
-      this.other = other;
-      this.composite = composite;
-    }
-
-    /** Default constructor, needed for hibernate. */
-    public TestStringEntity() {}
-  }
-
-  @Entity(name = "CompositeKeyTestStringEntity")
-  @com.googlecode.objectify.annotation.Entity
-  @WithStringVKey(classNameSuffix = "CompositeStringType", compositeKey = true)
-  static class CompositeKeyTestStringEntity {
-    @com.googlecode.objectify.annotation.Id @Id String id = "id";
-  }
-}

core/src/test/java/google/registry/persistence/converter/VKeyConverterTest.java

@@ -0,0 +1,103 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.testing.DatabaseHelper.insertInDb;
+
+import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithVKey;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Test SQL persistence of {@link VKey}. */
+public class VKeyConverterTest {
+
+  @RegisterExtension
+  public final JpaUnitTestExtension jpa =
+      new JpaTestExtensions.Builder()
+          .withoutCannedData()
+          .withEntityClass(
+              TestEntity.class,
+              TestStringEntity.class,
+              TestLongEntity.class,
+              VKeyConverter_TestStringEntity.class,
+              VKeyConverter_TestLongEntity.class)
+          .buildUnitTestExtension();
+
+  @Test
+  void testRoundTrip() {
+    TestStringEntity stringEntity = new TestStringEntity("TheRealSpartacus");
+    VKey<TestStringEntity> stringKey = VKey.createSql(TestStringEntity.class, "TheRealSpartacus");
+    TestLongEntity longEntity = new TestLongEntity(300L);
+    VKey<TestLongEntity> longKey = VKey.createSql(TestLongEntity.class, 300L);
+    TestEntity original = new TestEntity(1984L, stringKey, longKey);
+    insertInDb(stringEntity, longEntity, original);
+
+    TestEntity retrieved =
+        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestEntity.class, 1984L));
+    assertThat(retrieved.stringKey).isEqualTo(stringKey);
+    assertThat(retrieved.longKey).isEqualTo(longKey);
+  }
+
+  @Entity(name = "TestStringEntity")
+  @WithVKey(String.class)
+  protected static class TestStringEntity extends ImmutableObject {
+    @Id String id;
+
+    TestStringEntity(String id) {
+      this.id = id;
+    }
+
+    /** Default constructor, needed for hibernate. */
+    public TestStringEntity() {}
+  }
+
+  @Entity(name = "TestLongEntity")
+  @WithVKey(Long.class)
+  protected static class TestLongEntity extends ImmutableObject {
+    @Id Long id;
+
+    TestLongEntity(Long id) {
+      this.id = id;
+    }
+
+    /** Default constructor, needed for hibernate. */
+    public TestLongEntity() {}
+  }
+
+  @Entity(name = "TestEntity")
+  @WithVKey(String.class)
+  protected static class TestEntity extends ImmutableObject {
+    @Id Long id;
+    VKey<TestStringEntity> stringKey;
+    VKey<TestLongEntity> longKey;
+
+    TestEntity(Long id, VKey<TestStringEntity> stringKey, VKey<TestLongEntity> longKey) {
+      this.id = id;
+      this.stringKey = stringKey;
+      this.longKey = longKey;
+    }
+
+    /** Default constructor, needed for hibernate. */
+    public TestEntity() {}
+  }
+}

core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java

@@ -100,6 +100,24 @@ public class Spec11RegistrarThreatMatchesParserTest {
     assertThat(objectWithExtraFields).isEqualTo(objectWithoutExtraFields);
   }
 
+  @Test
+  void testSuccess_worksWithOutdatedField() throws Exception {
+    ThreatMatch objectWithOutdatedField =
+        ThreatMatch.fromJSON(
+            new JSONObject(
+                ImmutableMap.of(
+                    "threatType", "MALWARE",
+                    "fullyQualifiedDomainName", "c.com")));
+    ThreatMatch objectWithoutOutdatedFields =
+        ThreatMatch.fromJSON(
+            new JSONObject(
+                ImmutableMap.of(
+                    "threatType", "MALWARE",
+                    "domainName", "c.com")));
+
+    assertThat(objectWithOutdatedField).isEqualTo(objectWithoutOutdatedFields);
+  }
+
   /** The expected contents of the sample spec11 report file */
   public static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
     return ImmutableSet.of(getMatchA(), getMatchB());

core/src/test/java/google/registry/testing/AppEngineExtension.java

@@ -40,7 +40,7 @@ import com.google.common.collect.Sets;
 import com.google.common.io.Files;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.ObjectifyFilter;
-import google.registry.model.IdService;
+import google.registry.model.IdService.SelfAllocatedIdSupplier;
 import google.registry.model.ofy.ObjectifyService;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.State;
@@ -441,7 +441,7 @@ public final class AppEngineExtension implements BeforeEachCallback, AfterEachCa
 
     ObjectifyService.initOfy();
     // Reset id allocation in ObjectifyService so that ids are deterministic in tests.
-    IdService.resetSelfAllocatedId();
+    SelfAllocatedIdSupplier.getInstance().reset();
     this.ofyTestEntities.forEach(AppEngineExtension::register);
   }
 

core/src/test/java/google/registry/tools/CommandTestCase.java

@@ -19,7 +19,6 @@ import static com.google.common.collect.Iterables.toArray;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.JCommander;
 import com.google.common.base.Joiner;
@@ -63,7 +62,7 @@ public abstract class CommandTestCase<C extends Command> {
 
   protected C command;
 
-  protected final FakeClock fakeClock = new FakeClock(DateTime.now(UTC));
+  protected final FakeClock fakeClock = new FakeClock(DateTime.parse("2022-09-01T00:00:00.000Z"));
 
   @RegisterExtension
   public final AppEngineExtension appEngine =

core/src/test/java/google/registry/tools/GetContactCommandTest.java

@@ -19,22 +19,19 @@ import static google.registry.testing.DatabaseHelper.newContact;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
 import static google.registry.testing.DatabaseHelper.persistDeletedContact;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static org.joda.time.DateTimeZone.UTC;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.beust.jcommander.ParameterException;
-import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link GetContactCommand}. */
 class GetContactCommandTest extends CommandTestCase<GetContactCommand> {
 
-  private DateTime now = DateTime.now(UTC);
-
   @BeforeEach
   void beforeEach() {
     createTld("tld");
+    command.clock = fakeClock;
   }
 
   @Test
@@ -67,7 +64,7 @@ class GetContactCommandTest extends CommandTestCase<GetContactCommand> {
 
   @Test
   void testSuccess_deletedContact() throws Exception {
-    persistDeletedContact("sh8013", now.minusDays(1));
+    persistDeletedContact("sh8013", fakeClock.nowUtc().minusDays(1));
     runCommand("sh8013");
     assertInStdout("Contact 'sh8013' does not exist or is deleted");
   }
@@ -85,8 +82,9 @@ class GetContactCommandTest extends CommandTestCase<GetContactCommand> {
 
   @Test
   void testSuccess_contactDeletedInFuture() throws Exception {
-    persistResource(newContact("sh8013").asBuilder().setDeletionTime(now.plusDays(1)).build());
-    runCommand("sh8013", "--read_timestamp=" + now.plusMonths(1));
+    persistResource(
+        newContact("sh8013").asBuilder().setDeletionTime(fakeClock.nowUtc().plusDays(1)).build());
+    runCommand("sh8013", "--read_timestamp=" + fakeClock.nowUtc().plusMonths(1));
     assertInStdout("Contact 'sh8013' does not exist or is deleted");
   }
 }

core/src/test/java/google/registry/tools/GetDomainCommandTest.java

@@ -31,6 +31,7 @@ class GetDomainCommandTest extends CommandTestCase<GetDomainCommand> {
   @BeforeEach
   void beforeEach() {
     createTld("tld");
+    command.clock = fakeClock;
   }
 
   @Test

core/src/test/java/google/registry/tools/GetHostCommandTest.java

@@ -19,22 +19,19 @@ import static google.registry.testing.DatabaseHelper.newHost;
 import static google.registry.testing.DatabaseHelper.persistActiveHost;
 import static google.registry.testing.DatabaseHelper.persistDeletedHost;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static org.joda.time.DateTimeZone.UTC;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.beust.jcommander.ParameterException;
-import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link GetHostCommand}. */
 class GetHostCommandTest extends CommandTestCase<GetHostCommand> {
 
-  private DateTime now = DateTime.now(UTC);
-
   @BeforeEach
   void beforeEach() {
     createTld("tld");
+    command.clock = fakeClock;
   }
 
   @Test
@@ -77,7 +74,7 @@ class GetHostCommandTest extends CommandTestCase<GetHostCommand> {
 
   @Test
   void testSuccess_deletedHost() throws Exception {
-    persistDeletedHost("ns1.example.tld", now.minusDays(1));
+    persistDeletedHost("ns1.example.tld", fakeClock.nowUtc().minusDays(1));
     runCommand("ns1.example.tld");
     assertInStdout("Host 'ns1.example.tld' does not exist or is deleted");
   }
@@ -91,8 +88,11 @@ class GetHostCommandTest extends CommandTestCase<GetHostCommand> {
   @Test
   void testSuccess_hostDeletedInFuture() throws Exception {
     persistResource(
-        newHost("ns1.example.tld").asBuilder().setDeletionTime(now.plusDays(1)).build());
-    runCommand("ns1.example.tld", "--read_timestamp=" + now.plusMonths(1));
+        newHost("ns1.example.tld")
+            .asBuilder()
+            .setDeletionTime(fakeClock.nowUtc().plusDays(1))
+            .build());
+    runCommand("ns1.example.tld", "--read_timestamp=" + fakeClock.nowUtc().plusMonths(1));
     assertInStdout("Host 'ns1.example.tld' does not exist or is deleted");
   }
 

core/src/test/java/google/registry/tools/GetPackagePromotionCommandTest.java

@@ -0,0 +1,135 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import com.beust.jcommander.ParameterException;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.billing.BillingEvent.RenewalPriceBehavior;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.domain.token.AllocationToken.TokenType;
+import google.registry.model.domain.token.PackagePromotion;
+import org.joda.money.CurrencyUnit;
+import org.joda.money.Money;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link GetPackagePromotionCommand}. */
+public class GetPackagePromotionCommandTest extends CommandTestCase<GetPackagePromotionCommand> {
+
+  @BeforeEach
+  void beforeEach() {
+    command.clock = fakeClock;
+  }
+
+  @Test
+  void testSuccess() throws Exception {
+    AllocationToken token =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("abc123")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    PackagePromotion packagePromotion =
+        new PackagePromotion.Builder()
+            .setToken(token)
+            .setMaxDomains(100)
+            .setMaxCreates(500)
+            .setPackagePrice(Money.of(CurrencyUnit.USD, 1000))
+            .setNextBillingDate(DateTime.parse("2012-11-12T05:00:00Z"))
+            .setLastNotificationSent(DateTime.parse("2010-11-12T05:00:00Z"))
+            .build();
+    jpaTm().transact(() -> jpaTm().put(packagePromotion));
+    runCommand("abc123");
+  }
+
+  @Test
+  void testSuccessMultiplePackages() throws Exception {
+    AllocationToken token =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("abc123")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .put(
+                        new PackagePromotion.Builder()
+                            .setToken(token)
+                            .setMaxDomains(100)
+                            .setMaxCreates(500)
+                            .setPackagePrice(Money.of(CurrencyUnit.USD, 1000))
+                            .setNextBillingDate(DateTime.parse("2012-11-12T05:00:00Z"))
+                            .setLastNotificationSent(DateTime.parse("2010-11-12T05:00:00Z"))
+                            .build()));
+    AllocationToken token2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("123abc")
+                .setTokenType(TokenType.PACKAGE)
+                .setCreationTimeForTest(DateTime.parse("2012-11-12T05:00:00Z"))
+                .setAllowedTlds(ImmutableSet.of("foo"))
+                .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
+                .setRenewalPriceBehavior(RenewalPriceBehavior.SPECIFIED)
+                .setDiscountFraction(1)
+                .build());
+    jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .put(
+                        new PackagePromotion.Builder()
+                            .setToken(token2)
+                            .setMaxDomains(1000)
+                            .setMaxCreates(700)
+                            .setPackagePrice(Money.of(CurrencyUnit.USD, 3000))
+                            .setNextBillingDate(DateTime.parse("2014-11-12T05:00:00Z"))
+                            .setLastNotificationSent(DateTime.parse("2013-11-12T05:00:00Z"))
+                            .build()));
+
+    runCommand("abc123", "123abc");
+  }
+
+  @Test
+  void testFailure_packageDoesNotExist() {
+    IllegalArgumentException thrown =
+        assertThrows(IllegalArgumentException.class, () -> runCommand("fakeToken"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo("PackagePromotion with package token fakeToken does not exist");
+  }
+
+  @Test
+  void testFailure_noToken() {
+    assertThrows(ParameterException.class, this::runCommand);
+  }
+}

core/src/test/java/google/registry/tools/UniformRapidSuspensionCommandTest.java

@@ -48,6 +48,7 @@ class UniformRapidSuspensionCommandTest
 
   @BeforeEach
   void beforeEach() {
+    command.clock = fakeClock;
     // Since the command's history client ID must be CharlestonRoad, resave TheRegistrar that way.
     persistResource(
         loadRegistrar("TheRegistrar").asBuilder().setRegistrarId("CharlestonRoad").build());

core/src/test/java/google/registry/tools/javascrap/CreateSyntheticDomainHistoriesPipelineTest.java

@@ -1,78 +0,0 @@
-// Copyright 2022 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.testing.DatabaseHelper.createTld;
-import static google.registry.testing.DatabaseHelper.loadAllOf;
-import static google.registry.testing.DatabaseHelper.newDomain;
-import static google.registry.testing.DatabaseHelper.persistActiveHost;
-import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
-import static google.registry.testing.DatabaseHelper.persistSimpleResource;
-
-import com.google.common.collect.Iterables;
-import google.registry.beam.TestPipelineExtension;
-import google.registry.model.domain.Domain;
-import google.registry.model.domain.DomainHistory;
-import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.transaction.JpaTestExtensions;
-import google.registry.testing.DatastoreEntityExtension;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-/** Tests for {@link CreateSyntheticDomainHistoriesPipeline}. */
-public class CreateSyntheticDomainHistoriesPipelineTest {
-
-  @RegisterExtension
-  JpaTestExtensions.JpaIntegrationTestExtension jpaEextension =
-      new JpaTestExtensions.Builder().buildIntegrationTestExtension();
-
-  @RegisterExtension
-  DatastoreEntityExtension datastoreEntityExtension =
-      new DatastoreEntityExtension().allThreads(true);
-
-  @RegisterExtension TestPipelineExtension pipeline = TestPipelineExtension.create();
-
-  private Domain domain;
-
-  @BeforeEach
-  void beforeEach() {
-    persistNewRegistrar("TheRegistrar");
-    persistNewRegistrar("NewRegistrar");
-    createTld("tld");
-    domain =
-        persistSimpleResource(
-            newDomain("example.tld")
-                .asBuilder()
-                .setNameservers(persistActiveHost("external.com").createVKey())
-                .build());
-  }
-
-  @Test
-  void testSuccess() {
-    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(DomainHistory.class))).isEmpty();
-    CreateSyntheticDomainHistoriesPipeline.setup(pipeline, "NewRegistrar");
-    pipeline.run().waitUntilFinish();
-    DomainHistory domainHistory = Iterables.getOnlyElement(loadAllOf(DomainHistory.class));
-    assertThat(domainHistory.getType()).isEqualTo(HistoryEntry.Type.SYNTHETIC);
-    assertThat(domainHistory.getRegistrarId()).isEqualTo("NewRegistrar");
-    assertAboutImmutableObjects()
-        .that(domainHistory.getDomainBase().get())
-        .hasFieldsEqualTo(domain);
-  }
-}

core/src/test/resources/google/registry/flows/domain/domain_update_status_change.xml

@@ -5,12 +5,11 @@
        xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
         <domain:name>example.tld</domain:name>
         <domain:add>
-          <domain:contact type="admin">mak21</domain:contact>
-          <domain:contact type="billing">mak21</domain:contact>
-          <domain:contact type="tech">mak21</domain:contact>
-          <domain:status s="serverHold"
-              lang="en">Server hold.</domain:status>
+          <domain:status s="%STATUS_ADD%"/>
         </domain:add>
+        <domain:rem>
+          <domain:status s="%STATUS_REM%"/>
+        </domain:rem>
       </domain:update>
     </update>
     <clTRID>ABC-12345</clTRID>

docs/flows.md

@@ -754,7 +754,6 @@ new ones with the correct approval time).
     *   The allocation token is not valid for this registrar.
     *   The allocation token is not valid for this TLD.
     *   The allocation token was already redeemed.
-    *   The __REMOVEPACKAGE__ token is missing on a package domain command
 *   2306
     *   Domain transfer period must be one year.
     *   Domain transfer period must be zero or one year when using the superuser

java_common.gradle

@@ -62,6 +62,12 @@ configurations {
     // See https://issues.apache.org/jira/browse/BEAM-8862
     it.exclude group: 'org.mockito', module: 'mockito-core'
   }
+  all.each {
+    // log4j has high-profile security vulnerabilities. It's a transitive
+    // dependency used by some Apache Beam packages. Excluding it does not
+    // impact our troubleshooting needs.
+    it.exclude group: 'org.apache.logging.log4j'
+  }
 }
 
 dependencies {

processor/src/main/java/google/registry/processors/AbstractVKeyProcessor.java

@@ -1,185 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.processors;
-
-import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableList.toImmutableList;
-
-import com.squareup.javapoet.AnnotationSpec;
-import com.squareup.javapoet.ClassName;
-import com.squareup.javapoet.JavaFile;
-import com.squareup.javapoet.MethodSpec;
-import com.squareup.javapoet.ParameterizedTypeName;
-import com.squareup.javapoet.TypeName;
-import com.squareup.javapoet.TypeSpec;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import javax.annotation.processing.AbstractProcessor;
-import javax.annotation.processing.RoundEnvironment;
-import javax.lang.model.element.AnnotationMirror;
-import javax.lang.model.element.AnnotationValue;
-import javax.lang.model.element.Element;
-import javax.lang.model.element.ExecutableElement;
-import javax.lang.model.element.Modifier;
-import javax.lang.model.element.TypeElement;
-import javax.lang.model.type.DeclaredType;
-import javax.lang.model.type.TypeKind;
-import javax.lang.model.type.TypeMirror;
-import javax.lang.model.util.ElementFilter;
-import javax.lang.model.util.Elements;
-import javax.lang.model.util.Types;
-import javax.persistence.AttributeConverter;
-import javax.persistence.Converter;
-
-/** Abstract processor to generate {@link AttributeConverter} for VKey type. */
-public abstract class AbstractVKeyProcessor extends AbstractProcessor {
-
-  private static final String CONVERTER_CLASS_NAME_TEMP = "VKeyConverter_%s";
-  // The method with same name should be defined in WithStringVKey and WithLongVKey
-  private static final String CLASS_NAME_SUFFIX_KEY = "classNameSuffix";
-
-  // Method in WithStringVKey and WithLongVKey to indicate that this is a composite key.
-  private static final String COMPOSITE_KEY_KEY = "compositeKey";
-
-  abstract Class<?> getSqlColumnType();
-
-  abstract String getAnnotationSimpleName();
-
-  @Override
-  public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
-    annotations.forEach(
-        vKeyAnnotationType -> {
-          ElementFilter.typesIn(roundEnv.getElementsAnnotatedWith(vKeyAnnotationType))
-              .forEach(
-                  annotatedTypeElement -> {
-                    DeclaredType entityType = getDeclaredType(annotatedTypeElement);
-
-                    List<AnnotationMirror> actualAnnotation =
-                        annotatedTypeElement.getAnnotationMirrors().stream()
-                            .filter(
-                                annotationType ->
-                                    annotationType
-                                        .getAnnotationType()
-                                        .asElement()
-                                        .equals(vKeyAnnotationType))
-                            .collect(toImmutableList());
-                    checkState(
-                        actualAnnotation.size() == 1,
-                        String.format(
-                            "type can have only 1 %s annotation", getAnnotationSimpleName()));
-                    String converterClassNameSuffix = "";
-                    boolean hasCompositeOfyKey = false;
-                    for (Map.Entry<? extends ExecutableElement, ? extends AnnotationValue> entry :
-                        actualAnnotation.get(0).getElementValues().entrySet()) {
-                      String keyName = entry.getKey().getSimpleName().toString();
-                      Object value = entry.getValue().getValue();
-                      if (keyName.equals(CLASS_NAME_SUFFIX_KEY)) {
-                        converterClassNameSuffix = ((String) value).trim();
-                      } else if (keyName.equals(COMPOSITE_KEY_KEY)) {
-                        hasCompositeOfyKey = (Boolean) value;
-                      }
-                    }
-                    if (converterClassNameSuffix.isEmpty()) {
-                      converterClassNameSuffix =
-                          getTypeUtils().asElement(entityType).getSimpleName().toString();
-                    }
-
-                    try {
-                      createJavaFile(
-                              getPackageName(annotatedTypeElement),
-                              String.format(CONVERTER_CLASS_NAME_TEMP, converterClassNameSuffix),
-                              entityType,
-                              hasCompositeOfyKey)
-                          .writeTo(processingEnv.getFiler());
-                    } catch (IOException e) {
-                      throw new UncheckedIOException(e);
-                    }
-                  });
-        });
-    return false;
-  }
-
-  private JavaFile createJavaFile(
-      String packageName,
-      String converterClassName,
-      TypeMirror entityTypeMirror,
-      boolean hasCompositeOfyKey) {
-    TypeName entityType = ClassName.get(entityTypeMirror);
-
-    ParameterizedTypeName attributeConverter =
-        ParameterizedTypeName.get(
-            ClassName.get("google.registry.persistence.converter", "VKeyConverter"),
-            entityType,
-            ClassName.get(getSqlColumnType()));
-
-    MethodSpec getAttributeClass =
-        MethodSpec.methodBuilder("getAttributeClass")
-            .addAnnotation(Override.class)
-            .addModifiers(Modifier.PROTECTED)
-            .returns(
-                ParameterizedTypeName.get(
-                    ClassName.get(Class.class), ClassName.get(entityTypeMirror)))
-            .addStatement("return $T.class", entityType)
-            .build();
-
-    TypeSpec.Builder classBuilder =
-        TypeSpec.classBuilder(converterClassName)
-            .addAnnotation(
-                AnnotationSpec.builder(ClassName.get(Converter.class))
-                    .addMember("autoApply", "true")
-                    .build())
-            .addModifiers(Modifier.FINAL)
-            .superclass(attributeConverter)
-            .addMethod(getAttributeClass);
-
-    // If this is a converter for a composite vkey type, generate an override for the default
-    // {@link google.registry.persistence.VKeyConverter.hasCompositeOfyKey()} method, which returns
-    // false.
-    if (hasCompositeOfyKey) {
-      MethodSpec hasCompositeOfyKeyMethod =
-          MethodSpec.methodBuilder("hasCompositeOfyKey")
-              .addAnnotation(Override.class)
-              .addModifiers(Modifier.PROTECTED)
-              .returns(boolean.class)
-              .addStatement("return true", entityType)
-              .build();
-
-      classBuilder.addMethod(hasCompositeOfyKeyMethod);
-    }
-
-    TypeSpec vKeyConverter = classBuilder.build();
-    return JavaFile.builder(packageName, vKeyConverter).build();
-  }
-
-  private DeclaredType getDeclaredType(Element element) {
-    checkState(element.asType().getKind() == TypeKind.DECLARED, "element is not a DeclaredType");
-    return (DeclaredType) element.asType();
-  }
-
-  private String getPackageName(Element element) {
-    return getElementUtils().getPackageOf(element).getQualifiedName().toString();
-  }
-
-  private Elements getElementUtils() {
-    return processingEnv.getElementUtils();
-  }
-
-  private Types getTypeUtils() {
-    return processingEnv.getTypeUtils();
-  }
-}

processor/src/main/java/google/registry/processors/LongVKeyProcessor.java

@@ -1,37 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.processors;
-
-import javax.annotation.processing.SupportedAnnotationTypes;
-import javax.annotation.processing.SupportedSourceVersion;
-import javax.lang.model.SourceVersion;
-
-/** Annotation processor for entity that is annotated with WithLongVKey. */
-@SupportedAnnotationTypes("google.registry.persistence.WithLongVKey")
-@SupportedSourceVersion(SourceVersion.RELEASE_8)
-public class LongVKeyProcessor extends AbstractVKeyProcessor {
-
-  private static final String ANNOTATION_SIMPLE_NAME = "WithLongVKey";
-
-  @Override
-  Class<?> getSqlColumnType() {
-    return Long.class;
-  }
-
-  @Override
-  String getAnnotationSimpleName() {
-    return ANNOTATION_SIMPLE_NAME;
-  }
-}

processor/src/main/java/google/registry/processors/StringVKeyProcessor.java

@@ -1,37 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.processors;
-
-import javax.annotation.processing.SupportedAnnotationTypes;
-import javax.annotation.processing.SupportedSourceVersion;
-import javax.lang.model.SourceVersion;
-
-/** Annotation processor for entity that is annotated with WithStringVKey. */
-@SupportedAnnotationTypes("google.registry.persistence.WithStringVKey")
-@SupportedSourceVersion(SourceVersion.RELEASE_8)
-public class StringVKeyProcessor extends AbstractVKeyProcessor {
-
-  private static final String ANNOTATION_SIMPLE_NAME = "WithStringVKey";
-
-  @Override
-  Class<?> getSqlColumnType() {
-    return String.class;
-  }
-
-  @Override
-  String getAnnotationSimpleName() {
-    return ANNOTATION_SIMPLE_NAME;
-  }
-}

processor/src/main/java/google/registry/processors/VKeyProcessor.java

@@ -0,0 +1,169 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.processors;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
+import com.squareup.javapoet.AnnotationSpec;
+import com.squareup.javapoet.ClassName;
+import com.squareup.javapoet.JavaFile;
+import com.squareup.javapoet.MethodSpec;
+import com.squareup.javapoet.ParameterizedTypeName;
+import com.squareup.javapoet.TypeName;
+import com.squareup.javapoet.TypeSpec;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.annotation.processing.SupportedSourceVersion;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.AnnotationMirror;
+import javax.lang.model.element.AnnotationValue;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.ExecutableElement;
+import javax.lang.model.element.Modifier;
+import javax.lang.model.element.TypeElement;
+import javax.lang.model.type.DeclaredType;
+import javax.lang.model.type.TypeKind;
+import javax.lang.model.util.ElementFilter;
+import javax.lang.model.util.Elements;
+import javax.lang.model.util.Types;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** Processor to generate {@link AttributeConverter} for {@code VKey} type. */
+@SupportedAnnotationTypes("google.registry.persistence.WithVKey")
+@SupportedSourceVersion(SourceVersion.RELEASE_8)
+public class VKeyProcessor extends AbstractProcessor {
+
+  private static final String CONVERTER_CLASS_NAME_TEMP = "VKeyConverter_%s";
+
+  private static final String VKEY_TYPE_METHOD_NAME = "value";
+
+  @Override
+  public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
+    annotations.forEach(
+        vKeyAnnotationType ->
+            ElementFilter.typesIn(roundEnv.getElementsAnnotatedWith(vKeyAnnotationType))
+                .forEach(
+                    annotatedTypeElement -> {
+                      DeclaredType entityType = getDeclaredType(annotatedTypeElement);
+                      String simpleTypeName =
+                          getTypeUtils().asElement(entityType).getSimpleName().toString();
+                      List<AnnotationMirror> actualAnnotations =
+                          annotatedTypeElement.getAnnotationMirrors().stream()
+                              .filter(
+                                  annotationType ->
+                                      annotationType
+                                          .getAnnotationType()
+                                          .asElement()
+                                          .equals(vKeyAnnotationType))
+                              .collect(toImmutableList());
+                      checkState(
+                          actualAnnotations.size() == 1,
+                          "% can have only one @WithVKey annotation",
+                          simpleTypeName);
+                      TypeName keyType = null;
+                      for (Map.Entry<? extends ExecutableElement, ? extends AnnotationValue> entry :
+                          actualAnnotations.get(0).getElementValues().entrySet()) {
+                        String keyName = entry.getKey().getSimpleName().toString();
+                        if (keyName.equals(VKEY_TYPE_METHOD_NAME)) {
+                          try {
+                            keyType =
+                                TypeName.get(Class.forName(entry.getValue().getValue().toString()));
+                          } catch (ClassNotFoundException e) {
+                            throw new RuntimeException(
+                                String.format(
+                                    "VKey key class %s is not valid",
+                                    entry.getValue().getValue().toString()),
+                                e);
+                          }
+                        }
+                      }
+                      try {
+                        createJavaFile(
+                                getPackageName(annotatedTypeElement),
+                                String.format(CONVERTER_CLASS_NAME_TEMP, simpleTypeName),
+                                TypeName.get(entityType),
+                                keyType)
+                            .writeTo(processingEnv.getFiler());
+                      } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                      }
+                    }));
+    return false;
+  }
+
+  private static JavaFile createJavaFile(
+      String packageName, String converterClassName, TypeName entityType, TypeName keyType) {
+    ParameterizedTypeName attributeConverter =
+        ParameterizedTypeName.get(
+            ClassName.get("google.registry.persistence.converter", "VKeyConverter"),
+            entityType,
+            keyType);
+
+    MethodSpec getEntityClass =
+        MethodSpec.methodBuilder("getEntityClass")
+            .addAnnotation(Override.class)
+            .addModifiers(Modifier.PROTECTED)
+            .returns(ParameterizedTypeName.get(ClassName.get(Class.class), entityType))
+            .addStatement("return $T.class", entityType)
+            .build();
+
+    MethodSpec getKeyClass =
+        MethodSpec.methodBuilder("getKeyClass")
+            .addAnnotation(Override.class)
+            .addModifiers(Modifier.PROTECTED)
+            .returns(ParameterizedTypeName.get(ClassName.get(Class.class), keyType))
+            .addStatement("return $T.class", keyType)
+            .build();
+
+    TypeSpec.Builder classBuilder =
+        TypeSpec.classBuilder(converterClassName)
+            .addAnnotation(
+                AnnotationSpec.builder(ClassName.get(Converter.class))
+                    .addMember("autoApply", "true")
+                    .build())
+            .addModifiers(Modifier.FINAL)
+            .superclass(attributeConverter)
+            .addMethod(getEntityClass)
+            .addMethod(getKeyClass);
+
+    TypeSpec vKeyConverter = classBuilder.build();
+    return JavaFile.builder(packageName, vKeyConverter).build();
+  }
+
+  private static DeclaredType getDeclaredType(Element element) {
+    checkState(element.asType().getKind() == TypeKind.DECLARED, "element is not a DeclaredType");
+    return (DeclaredType) element.asType();
+  }
+
+  private String getPackageName(Element element) {
+    return getElementUtils().getPackageOf(element).getQualifiedName().toString();
+  }
+
+  private Elements getElementUtils() {
+    return processingEnv.getElementUtils();
+  }
+
+  private Types getTypeUtils() {
+    return processingEnv.getTypeUtils();
+  }
+}

processor/src/main/resources/META-INF/services/javax.annotation.processing.Processor

@@ -1,2 +1 @@
-google.registry.processors.StringVKeyProcessor
-google.registry.processors.LongVKeyProcessor
+google.registry.processors.VKeyProcessor

util/src/main/java/google/registry/util/SendEmailService.java

@@ -14,8 +14,12 @@
 
 package google.registry.util;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Iterables.toArray;
 
+import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.util.EmailMessage.Attachment;
 import java.io.IOException;
@@ -44,6 +48,10 @@ public class SendEmailService {
   private final Retrier retrier;
   private final TransportEmailSender transportEmailSender;
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final ImmutableSet<String> ALLOWED_ENVS =
+      ImmutableSet.of("PRODUCTION", "UNITTEST");
+
   @Inject
   SendEmailService(Retrier retrier, TransportEmailSender transportEmailSender) {
     this.retrier = retrier;
@@ -55,37 +63,50 @@ public class SendEmailService {
    * on transient failures.
    */
   public void sendEmail(EmailMessage emailMessage) {
-    retrier.callWithRetry(
-        () -> {
-          Message msg =
-              new MimeMessage(
-                  Session.getDefaultInstance(new Properties(), /* authenticator= */ null));
-          msg.setFrom(emailMessage.from());
-          msg.addRecipients(
-              RecipientType.TO, toArray(emailMessage.recipients(), InternetAddress.class));
-          msg.setSubject(emailMessage.subject());
+    if (!ALLOWED_ENVS.contains(
+        Ascii.toUpperCase(System.getProperty("google.registry.environment", "UNITTEST")))) {
+      logger.atInfo().log(
+          String.format(
+              "Email with subject %s would have been sent to recipients %s",
+              emailMessage.subject().substring(0, Math.min(emailMessage.subject().length(), 15)),
+              String.join(
+                  " , ",
+                  emailMessage.recipients().stream()
+                      .map(ia -> ia.toString())
+                      .collect(toImmutableSet()))));
+    } else {
+      retrier.callWithRetry(
+          () -> {
+            Message msg =
+                new MimeMessage(
+                    Session.getDefaultInstance(new Properties(), /* authenticator= */ null));
+            msg.setFrom(emailMessage.from());
+            msg.addRecipients(
+                RecipientType.TO, toArray(emailMessage.recipients(), InternetAddress.class));
+            msg.setSubject(emailMessage.subject());
 
-          Multipart multipart = new MimeMultipart();
-          BodyPart bodyPart = new MimeBodyPart();
-          bodyPart.setContent(
-              emailMessage.body(),
-              emailMessage.contentType().orElse(MediaType.PLAIN_TEXT_UTF_8).toString());
-          multipart.addBodyPart(bodyPart);
+            Multipart multipart = new MimeMultipart();
+            BodyPart bodyPart = new MimeBodyPart();
+            bodyPart.setContent(
+                emailMessage.body(),
+                emailMessage.contentType().orElse(MediaType.PLAIN_TEXT_UTF_8).toString());
+            multipart.addBodyPart(bodyPart);
 
-          if (emailMessage.attachment().isPresent()) {
-            Attachment attachment = emailMessage.attachment().get();
-            BodyPart attachmentPart = new MimeBodyPart();
-            attachmentPart.setContent(attachment.content(), attachment.contentType().toString());
-            attachmentPart.setFileName(attachment.filename());
-            multipart.addBodyPart(attachmentPart);
-          }
-          msg.addRecipients(RecipientType.BCC, toArray(emailMessage.bccs(), Address.class));
-          msg.addRecipients(RecipientType.CC, toArray(emailMessage.ccs(), Address.class));
-          msg.setContent(multipart);
-          msg.saveChanges();
-          transportEmailSender.sendMessage(msg);
-        },
-        IOException.class,
-        MessagingException.class);
+            if (emailMessage.attachment().isPresent()) {
+              Attachment attachment = emailMessage.attachment().get();
+              BodyPart attachmentPart = new MimeBodyPart();
+              attachmentPart.setContent(attachment.content(), attachment.contentType().toString());
+              attachmentPart.setFileName(attachment.filename());
+              multipart.addBodyPart(attachmentPart);
+            }
+            msg.addRecipients(RecipientType.BCC, toArray(emailMessage.bccs(), Address.class));
+            msg.addRecipients(RecipientType.CC, toArray(emailMessage.ccs(), Address.class));
+            msg.setContent(multipart);
+            msg.saveChanges();
+            transportEmailSender.sendMessage(msg);
+          },
+          IOException.class,
+          MessagingException.class);
+    }
   }
 }