Enable filtering across all test tasks (#311)

The segregated test targets in core break the --tests filter.  Fix this by
defining a "testFilter" property and creating the FilteringTest task type that
applies it to the property set by "--tests".

core/build.gradle

@@ -558,10 +558,38 @@ artifacts {
   testRuntime testJar
 }
 
-task fragileTest(type: Test) {
+/**
+ * We have to break out the test suites because some of the tests conflict
+ * with one another, but unfortunately this breaks the "--tests" flag.  The
+ * --tests flag only applies to the task named on the command line (usually
+ * just "test"), not for all tasks of type "Test".
+ *
+ * As a better solution, FilteringTest sets testNameIncludePatterns (the
+ * internal property that --tests sets) from the value of the "testFilter"
+ * property, allowing us to filter across all the tests in core without
+ * explicitly specifying a test task or causing errors because there are no
+ * matching tests in the main task.
+ *
+ * To use it, define "testFilter" to be a comma-separated collection of class
+ * names (wildcards are allowed):
+ *
+ *   ./gradlew test -P testFilter=*.FooBar,google.registry.tools.ShellCommandTest
+ */
+class FilteringTest extends Test {
+
+  void setTests(List<String> tests) {
+    // Common exclude pattern. See README in parent directory for explanation.
+    exclude "**/*TestCase.*", "**/*TestSuite.*"
+    include tests
+    if (project.testFilter) {
+      testNameIncludePatterns = project.testFilter.split(',')
+    }
+  }
+}
+
+task fragileTest(type: FilteringTest) {
   // Common exclude pattern. See README in parent directory for explanation.
-  exclude "**/*TestCase.*", "**/*TestSuite.*"
-  include fragileTestPatterns
+  tests = fragileTestPatterns
 
   if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") {
     exclude dockerIncompatibleTestPatterns
@@ -571,10 +599,8 @@ task fragileTest(type: Test) {
   forkEvery 1
 }
 
-task outcastTest(type: Test) {
-  // Common exclude pattern. See README in parent directory for explanation.
-  exclude "**/*TestCase.*", "**/*TestSuite.*"
-  include outcastTestPatterns
+task outcastTest(type: FilteringTest) {
+  tests = outcastTestPatterns
 
   // Sets the maximum number of test executors that may exist at the same time.
   maxParallelForks 5
@@ -630,10 +656,8 @@ task registryTool(type: JavaExec) {
   }
 }
 
-task generateGoldenImages(type: Test) {
-  // Common exclude pattern. See README in parent directory for explanation.
-  exclude "**/*TestCase.*", "**/*TestSuite.*"
-  include "**/webdriver/*"
+task generateGoldenImages(type: FilteringTest) {
+  tests = ["**/webdriver/*"]
 
   // Sets the maximum number of test executors that may exist at the same time.
   maxParallelForks 5

core/src/main/java/google/registry/model/tmch/ClaimsListDao.java

@@ -0,0 +1,72 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.tmch;
+
+import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import google.registry.schema.tmch.ClaimsList;
+import javax.persistence.EntityManager;
+
+/** Data access object for {@link ClaimsList}. */
+public class ClaimsListDao {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static void save(ClaimsList claimsList) {
+    jpaTm().transact(() -> jpaTm().getEntityManager().persist(claimsList));
+  }
+
+  /**
+   * Try to save the given {@link ClaimsList} into Cloud SQL. If the save fails, the error will be
+   * logged but no exception will be thrown.
+   *
+   * <p>This method is used during the dual-write phase of database migration as Datastore is still
+   * the authoritative database.
+   */
+  public static void trySave(ClaimsList claimsList) {
+    try {
+      ClaimsListDao.save(claimsList);
+      logger.atInfo().log(
+          "Inserted %,d claims into Cloud SQL, created at %s",
+          claimsList.getLabelsToKeys().size(), claimsList.getTmdbGenerationTime());
+    } catch (Throwable e) {
+      logger.atSevere().withCause(e).log("Error inserting claims into Cloud SQL");
+    }
+  }
+
+  /**
+   * Returns the current revision of the {@link ClaimsList} in Cloud SQL. Throws exception if there
+   * is no claims in the table.
+   */
+  public static ClaimsList getCurrent() {
+    return jpaTm()
+        .transact(
+            () -> {
+              EntityManager em = jpaTm().getEntityManager();
+              Long revisionId =
+                  em.createQuery("SELECT MAX(revisionId) FROM ClaimsList", Long.class)
+                      .getSingleResult();
+              return em.createQuery(
+                      "FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
+                          + " :revisionId",
+                      ClaimsList.class)
+                  .setParameter("revisionId", revisionId)
+                  .getSingleResult();
+            });
+  }
+
+  private ClaimsListDao() {}
+}

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -217,8 +217,7 @@ public class ClaimsListShard extends ImmutableObject {
             });
   }
 
-  public static ClaimsListShard create(
-      DateTime creationTime, ImmutableMap<String, String> labelsToKeys) {
+  public static ClaimsListShard create(DateTime creationTime, Map<String, String> labelsToKeys) {
     ClaimsListShard instance = new ClaimsListShard();
     instance.id = allocateId();
     instance.creationTime = checkNotNull(creationTime);

core/src/main/java/google/registry/model/transaction/JpaTransactionManagerImpl.java

@@ -15,10 +15,7 @@
 package google.registry.model.transaction;
 
 import com.google.common.flogger.FluentLogger;
-import google.registry.persistence.PersistenceModule.AppEngineEmf;
 import google.registry.util.Clock;
-import javax.inject.Inject;
-import javax.inject.Singleton;
 import javax.persistence.EntityManager;
 import javax.persistence.EntityManagerFactory;
 import javax.persistence.EntityTransaction;
@@ -26,7 +23,6 @@ import javax.persistence.PersistenceException;
 import org.joda.time.DateTime;
 
 /** Implementation of {@link JpaTransactionManager} for JPA compatible database. */
-@Singleton
 public class JpaTransactionManagerImpl implements JpaTransactionManager {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -39,8 +35,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   private final ThreadLocal<TransactionInfo> transactionInfo =
       ThreadLocal.withInitial(TransactionInfo::new);
 
-  @Inject
-  JpaTransactionManagerImpl(@AppEngineEmf EntityManagerFactory emf, Clock clock) {
+  public JpaTransactionManagerImpl(EntityManagerFactory emf, Clock clock) {
     this.emf = emf;
     this.clock = clock;
   }

core/src/main/java/google/registry/model/transaction/TransactionManagerFactory.java

@@ -49,6 +49,15 @@ public class TransactionManagerFactory {
     return new DatastoreTransactionManager(null);
   }
 
+  /**
+   * Sets jpaTm to the implementation for Nomulus tool. Note that this method should be only used by
+   * {@link google.registry.tools.RegistryCli} to initialize jpaTm.
+   */
+  public static void initForTool() {
+    // TODO(shicong): Uncomment the line below when we set up Cloud SQL instance in all environments
+    // jpaTm = DaggerPersistenceComponent.create().nomulusToolJpaTransactionManager();
+  }
+
   /** Returns {@link TransactionManager} instance. */
   public static TransactionManager tm() {
     return TM;
@@ -56,11 +65,6 @@ public class TransactionManagerFactory {
 
   /** Returns {@link JpaTransactionManager} instance. */
   public static JpaTransactionManager jpaTm() {
-    // TODO: Returns corresponding TransactionManager based on the runtime environment.
-    //  We have 3 kinds of runtime environment:
-    //    1. App Engine
-    //    2. Local JVM used by nomulus tool
-    //    3. Unit test
     return jpaTm;
   }
 }

core/src/main/java/google/registry/persistence/BloomFilterConverter.java

@@ -0,0 +1,59 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package google.registry.persistence;
+
+import static com.google.common.base.Charsets.US_ASCII;
+import static com.google.common.hash.Funnels.stringFunnel;
+
+import com.google.common.hash.BloomFilter;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import javax.annotation.Nullable;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA converter for ASCII String {@link BloomFilter}s. */
+@Converter(autoApply = true)
+public class BloomFilterConverter implements AttributeConverter<BloomFilter<String>, byte[]> {
+
+  @Override
+  @Nullable
+  public byte[] convertToDatabaseColumn(@Nullable BloomFilter<String> entity) {
+    if (entity == null) {
+      return null;
+    }
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    try {
+      entity.writeTo(bos);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Error saving Bloom filter data", e);
+    }
+    return bos.toByteArray();
+  }
+
+  @Override
+  @Nullable
+  public BloomFilter<String> convertToEntityAttribute(@Nullable byte[] columnValue) {
+    if (columnValue == null) {
+      return null;
+    }
+    try {
+      return BloomFilter.readFrom(new ByteArrayInputStream(columnValue), stringFunnel(US_ASCII));
+    } catch (IOException e) {
+      throw new UncheckedIOException("Error loading Bloom filter data", e);
+    }
+  }
+}

core/src/main/java/google/registry/persistence/PersistenceComponent.java

@@ -18,7 +18,9 @@ import dagger.Component;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.keyring.kms.KmsModule;
-import google.registry.model.transaction.JpaTransactionManagerImpl;
+import google.registry.model.transaction.JpaTransactionManager;
+import google.registry.persistence.PersistenceModule.AppEngineJpaTm;
+import google.registry.persistence.PersistenceModule.NomulusToolJpaTm;
 import google.registry.util.UtilsModule;
 import javax.inject.Singleton;
 import javax.persistence.EntityManagerFactory;
@@ -34,5 +36,10 @@ import javax.persistence.EntityManagerFactory;
       UtilsModule.class
     })
 public interface PersistenceComponent {
-  JpaTransactionManagerImpl jpaTransactionManager();
+
+  @AppEngineJpaTm
+  JpaTransactionManager appEngineJpaTransactionManager();
+
+  @NomulusToolJpaTm
+  JpaTransactionManager nomulusToolJpaTransactionManager();
 }

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -29,11 +29,14 @@ import dagger.Module;
 import dagger.Provides;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.kms.KmsKeyring;
+import google.registry.model.transaction.JpaTransactionManager;
+import google.registry.model.transaction.JpaTransactionManagerImpl;
+import google.registry.util.Clock;
 import java.lang.annotation.Documented;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
 import java.util.HashMap;
+import java.util.Map;
 import javax.inject.Qualifier;
+import javax.inject.Singleton;
 import javax.persistence.EntityManagerFactory;
 import javax.persistence.Persistence;
 import org.hibernate.cfg.Environment;
@@ -77,24 +80,55 @@ public class PersistenceModule {
   }
 
   @Provides
-  @AppEngineEmf
-  public static EntityManagerFactory providesAppEngineEntityManagerFactory(
+  @Singleton
+  @AppEngineJpaTm
+  public static JpaTransactionManager providesAppEngineJpaTm(
       @Config("cloudSqlJdbcUrl") String jdbcUrl,
       @Config("cloudSqlUsername") String username,
       @Config("cloudSqlInstanceConnectionName") String instanceConnectionName,
       KmsKeyring kmsKeyring,
-      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs) {
-    String password = kmsKeyring.getCloudSqlPassword();
-
+      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs,
+      Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
+
     // For Java users, the Cloud SQL JDBC Socket Factory can provide authenticated connections.
     // See https://github.com/GoogleCloudPlatform/cloud-sql-jdbc-socket-factory for details.
     overrides.put("socketFactory", "com.google.cloud.sql.postgres.SocketFactory");
     overrides.put("cloudSqlInstance", instanceConnectionName);
 
-    EntityManagerFactory emf = create(jdbcUrl, username, password, ImmutableMap.copyOf(overrides));
-    Runtime.getRuntime().addShutdownHook(new Thread(emf::close));
-    return emf;
+    overrides.put(Environment.URL, jdbcUrl);
+    overrides.put(Environment.USER, username);
+    overrides.put(Environment.PASS, kmsKeyring.getCloudSqlPassword());
+
+    return new JpaTransactionManagerImpl(create(overrides), clock);
+  }
+
+  @Provides
+  @Singleton
+  @NomulusToolJpaTm
+  public static JpaTransactionManager providesNomulusToolJpaTm(
+      @Config("toolsCloudSqlJdbcUrl") String jdbcUrl,
+      @Config("toolsCloudSqlUsername") String username,
+      @Config("cloudSqlInstanceConnectionName") String instanceConnectionName,
+      KmsKeyring kmsKeyring,
+      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs,
+      Clock clock) {
+
+    // Cloud SQL JDBC Socket Factory library requires the jdbc url to include all connection
+    // information, otherwise the connection initialization will fail. See here for more details:
+    // https://github.com/GoogleCloudPlatform/cloud-sql-jdbc-socket-factory
+    String fullJdbcUrl =
+        new StringBuilder(jdbcUrl)
+            .append("?cloudSqlInstance=" + instanceConnectionName)
+            .append("&socketFactory=com.google.cloud.sql.postgres.SocketFactory")
+            .append("&user=" + username)
+            .append("&password=" + kmsKeyring.getCloudSqlPassword())
+            .toString();
+
+    HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
+    overrides.put(Environment.URL, fullJdbcUrl);
+
+    return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
   /** Constructs the {@link EntityManagerFactory} instance. */
@@ -106,29 +140,36 @@ public class PersistenceModule {
     properties.put(Environment.USER, username);
     properties.put(Environment.PASS, password);
 
+    return create(ImmutableMap.copyOf(properties));
+  }
+
+  private static EntityManagerFactory create(Map<String, String> properties) {
     // If there are no annotated classes, we can create the EntityManagerFactory from the generic
     // method.  Otherwise we have to use a more tailored approach.  Note that this adds to the set
     // of annotated classes defined in the configuration, it does not override them.
     EntityManagerFactory emf =
-        Persistence.createEntityManagerFactory(PERSISTENCE_UNIT_NAME, properties);
-
+        Persistence.createEntityManagerFactory(
+            PERSISTENCE_UNIT_NAME, ImmutableMap.copyOf(properties));
     checkState(
         emf != null,
         "Persistence.createEntityManagerFactory() returns a null EntityManagerFactory");
     return emf;
   }
 
-  /** Dagger qualifier for the {@link EntityManagerFactory} used for App Engine application. */
+  /** Dagger qualifier for {@link JpaTransactionManager} used for App Engine application. */
   @Qualifier
   @Documented
-  @Retention(RetentionPolicy.RUNTIME)
-  public @interface AppEngineEmf {}
+  @interface AppEngineJpaTm {}
+
+  /** Dagger qualifier for {@link JpaTransactionManager} used for Nomulus tool. */
+  @Qualifier
+  @Documented
+  @interface NomulusToolJpaTm {}
 
   /** Dagger qualifier for the default Hibernate configurations. */
   // TODO(shicong): Change annotations in this class to none public or put them in a top level
   //  package
   @Qualifier
   @Documented
-  @Retention(RetentionPolicy.RUNTIME)
   public @interface DefaultHibernateConfigs {}
 }

core/src/main/java/google/registry/schema/tld/PremiumList.java

@@ -14,8 +14,12 @@
 
 package google.registry.schema.tld;
 
+import static com.google.common.base.Charsets.US_ASCII;
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.hash.Funnels.stringFunnel;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.BloomFilter;
 import google.registry.model.CreateAutoTimestamp;
 import java.math.BigDecimal;
 import java.util.Map;
@@ -67,11 +71,16 @@ public class PremiumList {
   @Column(name = "price", nullable = false)
   private Map<String, BigDecimal> labelsToPrices;
 
+  @Column(nullable = false)
+  private BloomFilter<String> bloomFilter;
+
   private PremiumList(String name, CurrencyUnit currency, Map<String, BigDecimal> labelsToPrices) {
-    // TODO(mcilwain): Generate the Bloom filter and set it here.
     this.name = name;
     this.currency = currency;
     this.labelsToPrices = labelsToPrices;
+    // ASCII is used for the charset because all premium list domain labels are stored punycoded.
+    this.bloomFilter = BloomFilter.create(stringFunnel(US_ASCII), labelsToPrices.size());
+    labelsToPrices.keySet().forEach(this.bloomFilter::put);
   }
 
   // Hibernate requires this default constructor.
@@ -101,7 +110,18 @@ public class PremiumList {
   }
 
   /** Returns a {@link Map} of domain labels to prices. */
-  public Map<String, BigDecimal> getLabelsToPrices() {
-    return labelsToPrices;
+  public ImmutableMap<String, BigDecimal> getLabelsToPrices() {
+    return ImmutableMap.copyOf(labelsToPrices);
+  }
+
+  /**
+   * Returns a Bloom filter to determine whether a label might be premium, or is definitely not.
+   *
+   * <p>If the domain label might be premium, then the next step is to check for the existence of a
+   * corresponding row in the PremiumListEntry table. Otherwise, we know for sure it's not premium,
+   * and no DB load is required.
+   */
+  public BloomFilter<String> getBloomFilter() {
+    return bloomFilter;
   }
 }

core/src/main/java/google/registry/schema/tmch/ClaimsList.java

@@ -15,7 +15,11 @@
 package google.registry.schema.tmch;
 
 import static com.google.common.base.Preconditions.checkState;
+import static google.registry.util.DateTimeUtils.toJodaDateTime;
+import static google.registry.util.DateTimeUtils.toZonedDateTime;
 
+import google.registry.model.CreateAutoTimestamp;
+import google.registry.model.ImmutableObject;
 import java.time.ZonedDateTime;
 import java.util.Map;
 import java.util.Optional;
@@ -29,6 +33,7 @@ import javax.persistence.Id;
 import javax.persistence.JoinColumn;
 import javax.persistence.MapKeyColumn;
 import javax.persistence.Table;
+import org.joda.time.DateTime;
 
 /**
  * A list of TMCH claims labels and their associated claims keys.
@@ -40,26 +45,29 @@ import javax.persistence.Table;
  * highest {@link #revisionId}.
  */
 @Entity
-@Table(name = "ClaimsList")
-public class ClaimsList {
+@Table
+public class ClaimsList extends ImmutableObject {
   @Id
   @GeneratedValue(strategy = GenerationType.IDENTITY)
-  @Column(name = "revision_id")
+  @Column
   private Long revisionId;
 
-  @Column(name = "creation_timestamp", nullable = false)
-  private ZonedDateTime creationTimestamp;
+  @Column(nullable = false)
+  private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
+
+  @Column(nullable = false)
+  private ZonedDateTime tmdbGenerationTime;
 
   @ElementCollection
   @CollectionTable(
       name = "ClaimsEntry",
-      joinColumns = @JoinColumn(name = "revision_id", referencedColumnName = "revision_id"))
-  @MapKeyColumn(name = "domain_label", nullable = false)
-  @Column(name = "claim_key", nullable = false)
+      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
+  @MapKeyColumn(name = "domainLabel", nullable = false)
+  @Column(name = "claimKey", nullable = false)
   private Map<String, String> labelsToKeys;
 
-  private ClaimsList(ZonedDateTime creationTimestamp, Map<String, String> labelsToKeys) {
-    this.creationTimestamp = creationTimestamp;
+  private ClaimsList(ZonedDateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
+    this.tmdbGenerationTime = tmdbGenerationTime;
     this.labelsToKeys = labelsToKeys;
   }
 
@@ -67,9 +75,8 @@ public class ClaimsList {
   private ClaimsList() {}
 
   /** Constructs a {@link ClaimsList} object. */
-  public static ClaimsList create(
-      ZonedDateTime creationTimestamp, Map<String, String> labelsToKeys) {
-    return new ClaimsList(creationTimestamp, labelsToKeys);
+  public static ClaimsList create(DateTime creationTimestamp, Map<String, String> labelsToKeys) {
+    return new ClaimsList(toZonedDateTime(creationTimestamp), labelsToKeys);
   }
 
   /** Returns the revision id of this claims list, or throws exception if it is null. */
@@ -79,9 +86,14 @@ public class ClaimsList {
     return revisionId;
   }
 
+  /** Returns the TMDB generation time of this claims list. */
+  public DateTime getTmdbGenerationTime() {
+    return toJodaDateTime(tmdbGenerationTime);
+  }
+
   /** Returns the creation time of this claims list. */
-  public ZonedDateTime getCreationTimestamp() {
-    return creationTimestamp;
+  public DateTime getCreationTimestamp() {
+    return creationTimestamp.getTimestamp();
   }
 
   /** Returns an {@link Map} mapping domain label to its lookup key. */

core/src/main/java/google/registry/tmch/ClaimsListParser.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableMap;
-import google.registry.model.tmch.ClaimsListShard;
+import google.registry.schema.tmch.ClaimsList;
 import java.util.List;
 import org.joda.time.DateTime;
 
@@ -34,11 +34,11 @@ import org.joda.time.DateTime;
 public class ClaimsListParser {
 
   /**
-   * Converts the lines from the DNL CSV file into a {@link ClaimsListShard} object.
+   * Converts the lines from the DNL CSV file into a {@link ClaimsList} object.
    *
    * <p>Please note that this does <b>not</b> insert the object into Datastore.
    */
-  public static ClaimsListShard parse(List<String> lines) {
+  public static ClaimsList parse(List<String> lines) {
     ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
 
     // First line: <version>,<DNL List creation datetime>
@@ -74,6 +74,6 @@ public class ClaimsListParser {
       builder.put(label, lookupKey);
     }
 
-    return ClaimsListShard.create(creationTime, builder.build());
+    return ClaimsList.create(creationTime, builder.build());
   }
 }

core/src/main/java/google/registry/tmch/TmchDnlAction.java

@@ -18,9 +18,11 @@ import static google.registry.request.Action.Method.POST;
 
 import com.google.common.flogger.FluentLogger;
 import google.registry.keyring.api.KeyModule.Key;
+import google.registry.model.tmch.ClaimsListDao;
 import google.registry.model.tmch.ClaimsListShard;
 import google.registry.request.Action;
 import google.registry.request.auth.Auth;
+import google.registry.schema.tmch.ClaimsList;
 import java.io.IOException;
 import java.security.SignatureException;
 import java.util.List;
@@ -54,10 +56,14 @@ public final class TmchDnlAction implements Runnable {
     } catch (SignatureException | IOException | PGPException e) {
       throw new RuntimeException(e);
     }
-    ClaimsListShard claims = ClaimsListParser.parse(lines);
-    claims.save();
+    ClaimsList claims = ClaimsListParser.parse(lines);
+    ClaimsListShard claimsListShard =
+        ClaimsListShard.create(claims.getTmdbGenerationTime(), claims.getLabelsToKeys());
+    claimsListShard.save();
     logger.atInfo().log(
         "Inserted %,d claims into Datastore, created at %s",
-        claims.size(), claims.getCreationTime());
+        claimsListShard.size(), claimsListShard.getCreationTime());
+
+    ClaimsListDao.trySave(claims);
   }
 }

core/src/main/java/google/registry/tools/CommandWithCloudSql.java

@@ -0,0 +1,23 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+/**
+ * Marker interface for commands that use Cloud Sql.
+ *
+ * <p>Just implementing this is sufficient to use Cloud Sql; {@link RegistryTool} will install it as
+ * needed.
+ */
+interface CommandWithCloudSql extends CommandWithRemoteApi {}

core/src/main/java/google/registry/tools/GenerateSqlSchemaCommand.java

@@ -19,34 +19,23 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableSet;
-import google.registry.model.domain.DesignatedContact;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.domain.GracePeriod;
-import google.registry.model.domain.secdns.DelegationSignerData;
-import google.registry.model.eppcommon.Trid;
-import google.registry.model.transfer.BaseTransferObject;
-import google.registry.model.transfer.TransferData;
-import google.registry.persistence.CreateAutoTimestampConverter;
 import google.registry.persistence.NomulusNamingStrategy;
 import google.registry.persistence.NomulusPostgreSQLDialect;
-import google.registry.persistence.UpdateAutoTimestampConverter;
-import google.registry.persistence.ZonedDateTimeConverter;
-import google.registry.schema.domain.RegistryLock;
-import google.registry.schema.tld.PremiumList;
-import google.registry.schema.tmch.ClaimsList;
+import google.registry.persistence.PersistenceModule;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import org.hibernate.boot.MetadataSources;
 import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
 import org.hibernate.cfg.Environment;
+import org.hibernate.jpa.boot.internal.ParsedPersistenceXmlDescriptor;
+import org.hibernate.jpa.boot.internal.PersistenceXmlParser;
 import org.hibernate.tool.hbm2ddl.SchemaExport;
 import org.hibernate.tool.schema.TargetType;
-import org.joda.time.Period;
 import org.testcontainers.containers.PostgreSQLContainer;
 
 /**
@@ -59,25 +48,6 @@ import org.testcontainers.containers.PostgreSQLContainer;
 @Parameters(separators = " =", commandDescription = "Generate PostgreSQL schema.")
 public class GenerateSqlSchemaCommand implements Command {
 
-  // TODO(mmuller): These should be read from persistence.xml so we don't need to maintain two
-  //                separate lists of all SQL table classes.
-  private static final ImmutableSet<Class> SQL_TABLE_CLASSES =
-      ImmutableSet.of(
-          BaseTransferObject.class,
-          ClaimsList.class,
-          CreateAutoTimestampConverter.class,
-          DelegationSignerData.class,
-          DesignatedContact.class,
-          DomainBase.class,
-          GracePeriod.class,
-          Period.class,
-          PremiumList.class,
-          RegistryLock.class,
-          TransferData.class,
-          Trid.class,
-          UpdateAutoTimestampConverter.class,
-          ZonedDateTimeConverter.class);
-
   @VisibleForTesting
   public static final String DB_OPTIONS_CLASH =
       "Database host and port may not be specified along with the option to start a "
@@ -164,7 +134,9 @@ public class GenerateSqlSchemaCommand implements Command {
 
       MetadataSources metadata =
           new MetadataSources(new StandardServiceRegistryBuilder().applySettings(settings).build());
-      SQL_TABLE_CLASSES.forEach(metadata::addAnnotatedClass);
+
+      addAnnotatedClasses(metadata, settings);
+
       SchemaExport schemaExport = new SchemaExport();
       schemaExport.setHaltOnError(true);
       schemaExport.setFormat(true);
@@ -203,4 +175,30 @@ public class GenerateSqlSchemaCommand implements Command {
       }
     }
   }
+
+  private void addAnnotatedClasses(MetadataSources metadata, Map<String, String> settings) {
+    ParsedPersistenceXmlDescriptor descriptor =
+        PersistenceXmlParser.locatePersistenceUnits(settings).stream()
+            .filter(unit -> PersistenceModule.PERSISTENCE_UNIT_NAME.equals(unit.getName()))
+            .findFirst()
+            .orElseThrow(
+                () ->
+                    new IllegalArgumentException(
+                        String.format(
+                            "Could not find persistence unit with name %s",
+                            PersistenceModule.PERSISTENCE_UNIT_NAME)));
+
+    List<String> classNames = descriptor.getManagedClassNames();
+    for (String className : classNames) {
+      try {
+        Class<?> clazz = Class.forName(className);
+        metadata.addAnnotatedClass(clazz);
+      } catch (ClassNotFoundException e) {
+        throw new IllegalArgumentException(
+            String.format(
+                "Could not load class with name %s present in persistence.xml", className),
+            e);
+      }
+    }
+  }
 }

core/src/main/java/google/registry/tools/RegistryCli.java

@@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import google.registry.config.RegistryConfig;
 import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.transaction.TransactionManagerFactory;
 import google.registry.tools.AuthModule.LoginRequiredException;
 import google.registry.tools.params.ParameterFactory;
 import java.io.ByteArrayInputStream;
@@ -210,6 +211,8 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
     }
 
     // CommandWithRemoteApis need to have the remote api installed to work.
+    // CommandWithCloudSql extends CommandWithRemoteApi so the command will also get the remote
+    // api installed. This is because the DB password is stored in Datastore.
     if (command instanceof CommandWithRemoteApi) {
       if (installer == null) {
         installer = new RemoteApiInstaller();
@@ -233,6 +236,10 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
       ofy().clearSessionCache();
     }
 
+    if (command instanceof CommandWithCloudSql) {
+      TransactionManagerFactory.initForTool();
+    }
+
     command.run();
   }
 

core/src/main/java/google/registry/tools/UploadClaimsListCommand.java

@@ -21,7 +21,9 @@ import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.common.base.Joiner;
 import com.google.common.io.Files;
+import google.registry.model.tmch.ClaimsListDao;
 import google.registry.model.tmch.ClaimsListShard;
+import google.registry.schema.tmch.ClaimsList;
 import google.registry.tmch.ClaimsListParser;
 import java.io.File;
 import java.io.IOException;
@@ -35,9 +37,14 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
   @Parameter(description = "Claims list filename")
   private List<String> mainParameters = new ArrayList<>();
 
+  @Parameter(
+      names = {"--also_cloud_sql"},
+      description = "Persist claims list to Cloud SQL in addition to Datastore; defaults to false.")
+  boolean alsoCloudSql;
+
   private String claimsListFilename;
 
-  private ClaimsListShard claimsList;
+  private ClaimsList claimsList;
 
   @Override
   protected void init() throws IOException {
@@ -56,7 +63,10 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command
 
   @Override
   public String execute() {
-    claimsList.save();
+    ClaimsListShard.create(claimsList.getTmdbGenerationTime(), claimsList.getLabelsToKeys()).save();
+    if (alsoCloudSql) {
+      ClaimsListDao.trySave(claimsList);
+    }
     return String.format("Successfully uploaded claims list %s", claimsListFilename);
   }
 }

core/src/main/resources/META-INF/persistence.xml

@@ -33,6 +33,7 @@
     <class>google.registry.model.eppcommon.Trid</class>
 
     <!-- Customized type converters -->
+    <class>google.registry.persistence.BloomFilterConverter</class>
     <class>google.registry.persistence.CreateAutoTimestampConverter</class>
     <class>google.registry.persistence.UpdateAutoTimestampConverter</class>
     <class>google.registry.persistence.ZonedDateTimeConverter</class>

core/src/test/java/google/registry/model/registry/RegistryLockDaoTest.java

@@ -19,7 +19,6 @@ import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.testing.JUnitBackports.assertThrows;
 
 import google.registry.model.transaction.JpaTransactionManagerRule;
-import google.registry.persistence.CreateAutoTimestampConverter;
 import google.registry.schema.domain.RegistryLock;
 import google.registry.schema.domain.RegistryLock.Action;
 import google.registry.testing.AppEngineRule;
@@ -38,9 +37,7 @@ public final class RegistryLockDaoTest {
 
   @Rule
   public final JpaTransactionManagerRule jpaTmRule =
-      new JpaTransactionManagerRule.Builder()
-          .withEntityClass(RegistryLock.class, CreateAutoTimestampConverter.class)
-          .build();
+      new JpaTransactionManagerRule.Builder().build();
 
   @Test
   public void testSaveAndLoad_success() {

core/src/test/java/google/registry/model/tmch/ClaimsListDaoTest.java

@@ -0,0 +1,95 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.tmch;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.JUnitBackports.assertThrows;
+
+import com.google.common.collect.ImmutableMap;
+import google.registry.model.transaction.JpaTransactionManagerRule;
+import google.registry.schema.tmch.ClaimsList;
+import google.registry.testing.FakeClock;
+import javax.persistence.NoResultException;
+import javax.persistence.PersistenceException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link ClaimsListDao}. */
+@RunWith(JUnit4.class)
+public class ClaimsListDaoTest {
+
+  private FakeClock fakeClock = new FakeClock();
+
+  @Rule
+  public final JpaTransactionManagerRule jpaTmRule =
+      new JpaTransactionManagerRule.Builder().build();
+
+  @Test
+  public void trySave_insertsClaimsListSuccessfully() {
+    ClaimsList claimsList =
+        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsListDao.trySave(claimsList);
+    ClaimsList insertedClaimsList = ClaimsListDao.getCurrent();
+    assertClaimsListEquals(claimsList, insertedClaimsList);
+    assertThat(insertedClaimsList.getCreationTimestamp())
+        .isEqualTo(jpaTmRule.getTxnClock().nowUtc());
+  }
+
+  @Test
+  public void trySave_noExceptionThrownWhenSaveFail() {
+    ClaimsList claimsList =
+        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsListDao.trySave(claimsList);
+    ClaimsList insertedClaimsList = ClaimsListDao.getCurrent();
+    assertClaimsListEquals(claimsList, insertedClaimsList);
+    // Save ClaimsList with existing revisionId should fail because revisionId is the primary key.
+    ClaimsListDao.trySave(insertedClaimsList);
+  }
+
+  @Test
+  public void trySave_claimsListWithNoEntries() {
+    ClaimsList claimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of());
+    ClaimsListDao.trySave(claimsList);
+    ClaimsList insertedClaimsList = ClaimsListDao.getCurrent();
+    assertClaimsListEquals(claimsList, insertedClaimsList);
+    assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty();
+  }
+
+  @Test
+  public void getCurrent_throwsNoResultExceptionIfTableIsEmpty() {
+    PersistenceException thrown =
+        assertThrows(PersistenceException.class, () -> ClaimsListDao.getCurrent());
+    assertThat(thrown).hasCauseThat().isInstanceOf(NoResultException.class);
+  }
+
+  @Test
+  public void getCurrent_returnsLatestClaims() {
+    ClaimsList oldClaimsList =
+        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsList newClaimsList =
+        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
+    ClaimsListDao.trySave(oldClaimsList);
+    ClaimsListDao.trySave(newClaimsList);
+    assertClaimsListEquals(newClaimsList, ClaimsListDao.getCurrent());
+  }
+
+  private void assertClaimsListEquals(ClaimsList left, ClaimsList right) {
+    assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId());
+    assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime());
+    assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys());
+  }
+}

core/src/test/java/google/registry/model/transaction/JpaTransactionManagerRule.java

@@ -27,9 +27,10 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import javax.persistence.EntityManagerFactory;
-import org.hibernate.boot.MetadataSources;
-import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
 import org.hibernate.cfg.Environment;
+import org.hibernate.jpa.boot.internal.ParsedPersistenceXmlDescriptor;
+import org.hibernate.jpa.boot.internal.PersistenceXmlParser;
+import org.hibernate.jpa.boot.spi.Bootstrap;
 import org.joda.time.DateTime;
 import org.junit.rules.ExternalResource;
 import org.junit.rules.RuleChain;
@@ -120,10 +121,19 @@ public class JpaTransactionManagerRule extends ExternalResource {
     properties.put(Environment.USER, username);
     properties.put(Environment.PASS, password);
 
-    MetadataSources metadataSources =
-        new MetadataSources(new StandardServiceRegistryBuilder().applySettings(properties).build());
-    extraEntityClasses.forEach(metadataSources::addAnnotatedClass);
-    return metadataSources.buildMetadata().getSessionFactoryBuilder().build();
+    ParsedPersistenceXmlDescriptor descriptor =
+        PersistenceXmlParser.locatePersistenceUnits(properties).stream()
+            .filter(unit -> PersistenceModule.PERSISTENCE_UNIT_NAME.equals(unit.getName()))
+            .findFirst()
+            .orElseThrow(
+                () ->
+                    new IllegalArgumentException(
+                        String.format(
+                            "Could not find persistence unit with name %s",
+                            PersistenceModule.PERSISTENCE_UNIT_NAME)));
+
+    extraEntityClasses.stream().map(Class::getName).forEach(descriptor::addClasses);
+    return Bootstrap.getEntityManagerFactoryBuilder(descriptor, properties).build();
   }
 
   /** Returns the {@link FakeClock} used by the underlying {@link JpaTransactionManagerImpl}. */

core/src/test/java/google/registry/persistence/BloomFilterConverterTest.java

@@ -0,0 +1,67 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package google.registry.persistence;
+
+import static com.google.common.base.Charsets.US_ASCII;
+import static com.google.common.hash.Funnels.stringFunnel;
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.hash.BloomFilter;
+import google.registry.model.ImmutableObject;
+import google.registry.model.transaction.JpaTransactionManagerRule;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import org.hibernate.cfg.Environment;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class BloomFilterConverterTest {
+
+  @Rule
+  public final JpaTransactionManagerRule jpaTmRule =
+      new JpaTransactionManagerRule.Builder()
+          .withEntityClass(TestEntity.class)
+          .withProperty(Environment.HBM2DDL_AUTO, "update")
+          .build();
+
+  @Test
+  public void roundTripConversion_returnsSameBloomFilter() {
+    BloomFilter<String> bloomFilter = BloomFilter.create(stringFunnel(US_ASCII), 3);
+    ImmutableSet.of("foo", "bar", "baz").forEach(bloomFilter::put);
+    TestEntity entity = new TestEntity(bloomFilter);
+    jpaTm().transact(() -> jpaTm().getEntityManager().persist(entity));
+    TestEntity persisted =
+        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestEntity.class, "id"));
+    assertThat(persisted.bloomFilter).isEqualTo(bloomFilter);
+  }
+
+  @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference.
+  public static class TestEntity extends ImmutableObject {
+
+    @Id String name = "id";
+
+    BloomFilter<String> bloomFilter;
+
+    public TestEntity() {}
+
+    public TestEntity(BloomFilter<String> bloomFilter) {
+      this.bloomFilter = bloomFilter;
+    }
+  }
+}

core/src/test/java/google/registry/persistence/CreateAutoTimestampConverterTest.java

@@ -23,32 +23,21 @@ import javax.persistence.Entity;
 import javax.persistence.Id;
 import org.hibernate.cfg.Environment;
 import org.joda.time.DateTime;
-import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
-import org.testcontainers.containers.PostgreSQLContainer;
 
 @RunWith(JUnit4.class)
 public class CreateAutoTimestampConverterTest {
 
-  @ClassRule
-  public static PostgreSQLContainer postgres =
-      new PostgreSQLContainer()
-          .withDatabaseName("postgres")
-          .withUsername("postgres")
-          .withPassword("domain-registry");
-
   @Rule
   public final JpaTransactionManagerRule jpaTmRule =
       new JpaTransactionManagerRule.Builder()
-          .withEntityClass(TestEntity.class, CreateAutoTimestampConverter.class)
+          .withEntityClass(TestEntity.class)
           .withProperty(Environment.HBM2DDL_AUTO, "update")
           .build();
 
-  public CreateAutoTimestampConverterTest() {}
-
   @Test
   public void testTypeConversion() {
     CreateAutoTimestamp ts = CreateAutoTimestamp.create(DateTime.parse("2019-09-9T11:39:00Z"));

core/src/test/java/google/registry/persistence/UpdateAutoTimestampConverterTest.java

@@ -22,32 +22,21 @@ import google.registry.model.transaction.JpaTransactionManagerRule;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import org.hibernate.cfg.Environment;
-import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
-import org.testcontainers.containers.PostgreSQLContainer;
 
 @RunWith(JUnit4.class)
 public class UpdateAutoTimestampConverterTest {
 
-  @ClassRule
-  public static PostgreSQLContainer postgres =
-      new PostgreSQLContainer()
-          .withDatabaseName("postgres")
-          .withUsername("postgres")
-          .withPassword("domain-registry");
-
   @Rule
   public final JpaTransactionManagerRule jpaTmRule =
       new JpaTransactionManagerRule.Builder()
-          .withEntityClass(TestEntity.class, UpdateAutoTimestampConverter.class)
+          .withEntityClass(TestEntity.class)
           .withProperty(Environment.HBM2DDL_AUTO, "update")
           .build();
 
-  public UpdateAutoTimestampConverterTest() {}
-
   @Test
   public void testTypeConversion() {
     TestEntity ent = new TestEntity("myinst", null);

core/src/test/java/google/registry/persistence/ZonedDateTimeConverterTest.java

@@ -37,7 +37,7 @@ public class ZonedDateTimeConverterTest {
   @Rule
   public final JpaTransactionManagerRule jpaTmRule =
       new JpaTransactionManagerRule.Builder()
-          .withEntityClass(TestEntity.class, ZonedDateTimeConverter.class)
+          .withEntityClass(TestEntity.class)
           .withProperty(Environment.HBM2DDL_AUTO, "update")
           .build();
 

core/src/test/java/google/registry/schema/tld/PremiumListDaoTest.java

@@ -20,7 +20,6 @@ import static google.registry.testing.JUnitBackports.assertThrows;
 
 import com.google.common.collect.ImmutableMap;
 import google.registry.model.transaction.JpaTransactionManagerRule;
-import google.registry.persistence.CreateAutoTimestampConverter;
 import java.math.BigDecimal;
 import javax.persistence.PersistenceException;
 import org.joda.money.CurrencyUnit;
@@ -35,9 +34,7 @@ public class PremiumListDaoTest {
 
   @Rule
   public final JpaTransactionManagerRule jpaTmRule =
-      new JpaTransactionManagerRule.Builder()
-          .withEntityClass(PremiumList.class, CreateAutoTimestampConverter.class)
-          .build();
+      new JpaTransactionManagerRule.Builder().build();
 
   private static final ImmutableMap<String, BigDecimal> TEST_PRICES =
       ImmutableMap.of(

core/src/test/java/google/registry/schema/tld/PremiumListTest.java

@@ -0,0 +1,50 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.schema.tld;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.hash.BloomFilter;
+import java.math.BigDecimal;
+import org.joda.money.CurrencyUnit;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link PremiumList}. */
+@RunWith(JUnit4.class)
+public class PremiumListTest {
+
+  private static final ImmutableMap<String, BigDecimal> TEST_PRICES =
+      ImmutableMap.of(
+          "silver",
+          BigDecimal.valueOf(10.23),
+          "gold",
+          BigDecimal.valueOf(1305.47),
+          "palladium",
+          BigDecimal.valueOf(1552.78));
+
+  @Test
+  public void bloomFilter_worksCorrectly() {
+    BloomFilter<String> bloomFilter =
+        PremiumList.create("testname", CurrencyUnit.USD, TEST_PRICES).getBloomFilter();
+    ImmutableSet.of("silver", "gold", "palladium")
+        .forEach(l -> assertThat(bloomFilter.mightContain(l)).isTrue());
+    ImmutableSet.of("dirt", "pyrite", "zirconia")
+        .forEach(l -> assertThat(bloomFilter.mightContain(l)).isFalse());
+  }
+}

db/README.md

@@ -3,6 +3,23 @@
 This project contains Nomulus's Cloud SQL schema and schema-deployment
 utilities.
 
+### Database Roles and Privileges
+
+Nomulus uses the 'postgres' database in the 'public' schema. The following
+users/roles are defined:
+
+*   postgres: the initial user is used for admin and schema deployment.
+    *  In Cloud SQL, we do not control superusers. The initial 'postgres' user
+       is a regular user with create-role/create-db privileges. Therefore,
+       it is not possible to separate admin user and schema-deployment user.
+*   readwrite is a role with read-write privileges on all data tables and
+    sequences. However, it does not have write access to admin tables. Nor
+    can it create new tables.
+    *   The Registry server user is granted this role.
+*   readonly is a role with SELECT privileges on all tables.
+    *   Reporting job user and individual human readers may be granted
+        this role.
+
 ### Schema DDL Scripts
 
 Currently we use Flyway for schema deployment. Versioned incremental update

db/build.gradle

@@ -12,18 +12,45 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import org.gradle.api.internal.tasks.userinput.UserInputHandler
+
 plugins {
   id "org.flywaydb.flyway" version "6.0.1"
   id 'maven-publish'
 }
 
 ext {
+  Set restrictedDbEnv =
+      [ 'sandbox', 'production' ].asUnmodifiable()
+  Set allDbEnv =
+      [ 'alpha', 'crash' ].plus(restrictedDbEnv).asUnmodifiable()
+
   def dbServerProperty = 'dbServer'
   def dbNameProperty = 'dbName'
 
-  def dbServer = findProperty(dbServerProperty)
+  def dbServer = findProperty(dbServerProperty).toString().toLowerCase()
   def dbName = findProperty(dbNameProperty)
 
+  reconfirmRestrictedDbEnv = {
+    if (!restrictedDbEnv.contains(dbServer)) {
+      return
+    }
+    // For restricted environments, ask the user to type again to confirm.
+    // The following statement uses Gradle internal API to get around the
+    // missing console bug when Gradle Daemon is in use. Another option is
+    // to use the ant.input task. For details please refer to
+    // https://github.com/gradle/gradle/issues/1251.
+    def dbServerAgain = services.get(UserInputHandler.class).askQuestion(
+        """\
+           Are you sure? Operating on ${dbServer} from desktop is unsafe.
+           Please type '${dbServer}' again to proceed: """.stripIndent(),
+        '').trim()
+    if (dbServer != dbServerAgain) {
+      throw new RuntimeException(
+          "Failed to confirm for restricted database environment. Operation aborted.")
+    }
+  }
+
   getAccessInfoByHostPort = { hostAndPort ->
     return [
         url: "jdbc:postgresql://${hostAndPort}/${dbName}",
@@ -31,8 +58,8 @@ ext {
         password: findProperty('dbPassword')]
   }
 
-  getSocketFactoryAccessInfo = {
-    def cred = getCloudSqlCredential('alpha', 'superuser').split(' ')
+  getSocketFactoryAccessInfo = { env ->
+    def cred = getCloudSqlCredential(env, 'admin').split(' ')
     def sqlInstance = cred[0]
     return [
         url: """\
@@ -46,11 +73,10 @@ ext {
   }
 
   getJdbcAccessInfo = {
-    switch (dbServer.toString().toLowerCase()) {
-      case 'alpha':
-        return getSocketFactoryAccessInfo()
-      default:
-        return getAccessInfoByHostPort(dbServer)
+    if (allDbEnv.contains(dbServer)) {
+      return getSocketFactoryAccessInfo(dbServer)
+    } else {
+      return getAccessInfoByHostPort(dbServer)
     }
   }
 
@@ -62,13 +88,16 @@ ext {
   // later).
   getCloudSqlCredential = { env, role ->
     env = env == 'production' ? '' : "-${env}"
+    def keyProject = env == '-crash'
+        ? 'domain-registry-crash-kms-keys'
+        : "domain-registry${env}-keys"
     def command =
         """gsutil cp \
-           gs://domain-registry${env}-cloudsql-credentials/${role}.enc - | \
+           gs://domain-registry${env}-cloudsql-credentials/${role}_credential.enc - | \
            gcloud kms decrypt --location global --keyring nomulus \
            --key sql-credentials-on-gcs-key --plaintext-file=- \
            --ciphertext-file=- \
-           --project=domain-registry${env}-keys"""
+           --project=${keyProject}"""
 
     return execInBash(command, '/tmp')
   }
@@ -114,6 +143,13 @@ flyway {
   locations = [ "classpath:sql/flyway" ]
 }
 
+tasks.flywayMigrate.dependsOn(
+    tasks.create('confirmMigrateOnRestrictedDb') {
+      doLast {
+        project.ext.reconfirmRestrictedDbEnv()
+      }
+    })
+
 dependencies {
   def deps = rootProject.dependencyMap
 

db/src/main/resources/sql/flyway/V6__premium_list_bloom_filter.sql

@@ -0,0 +1,15 @@
+-- Copyright 2019 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "PremiumList" add column if not exists bloom_filter bytea not null;

db/src/main/resources/sql/flyway/V7__update_claims_list.sql

@@ -0,0 +1,15 @@
+-- Copyright 2019 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "ClaimsList" add column if not exists tmdb_generation_time timestamp with time zone not null;

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -130,6 +130,7 @@
 
     create table "PremiumList" (
        revision_id  bigserial not null,
+        bloom_filter bytea not null,
         creation_timestamp timestamptz not null,
         currency bytea not null,
         name text not null,

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -50,7 +50,8 @@ CREATE TABLE public."ClaimsEntry" (
 
 CREATE TABLE public."ClaimsList" (
     revision_id bigint NOT NULL,
-    creation_timestamp timestamp with time zone NOT NULL
+    creation_timestamp timestamp with time zone NOT NULL,
+    tmdb_generation_time timestamp with time zone NOT NULL
 );
 
 
@@ -92,7 +93,8 @@ CREATE TABLE public."PremiumList" (
     revision_id bigint NOT NULL,
     creation_timestamp timestamp with time zone NOT NULL,
     currency bytea NOT NULL,
-    name text NOT NULL
+    name text NOT NULL,
+    bloom_filter bytea NOT NULL
 );
 
 

db/src/main/resources/sql/user/create_readonly_user.sql

@@ -12,10 +12,12 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
--- Script to create a user with read-only permission to all tables.
+-- Script to create a user with read-only permission to all tables. The
+-- initialize_roles.sql script creates the readonly role used here.
 
+-- Comment out line below if user already exists:
 CREATE USER :username ENCRYPTED PASSWORD :'password';
-GRANT CONNECT ON DATABASE postgres TO :username;
-GRANT USAGE ON SCHEMA public TO :username;
-GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO :username;
-GRANT SELECT ON ALL TABLES IN SCHEMA public TO :username;
+-- Comment out line above and uncomment line below if user has been created
+-- from Cloud Dashboard:
+-- ALTER USER :username NOCREATEDB NOCREATEROLE;
+GRANT readonly TO :username;

db/src/main/resources/sql/user/create_readwrite_user.sql

@@ -13,11 +13,12 @@
 -- limitations under the License.
 --
 -- Script to create a user with read-write permission to all tables (except for
--- WRITE permissions to flyway_schema_history).
+-- WRITE permissions to flyway_schema_history). The initialize_roles.sql script
+-- creates the readwrite role used here.
 
+-- Comment out line below if user already exists:
 CREATE USER :username ENCRYPTED PASSWORD :'password';
-GRANT CONNECT ON DATABASE postgres TO :username;
-GRANT USAGE ON SCHEMA public TO :username;
-GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO :username;
-GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO :username;
-REVOKE INSERT, UPDATE, DELETE ON TABLE public.flyway_schema_history FROM :username;
+-- Comment out line above and uncomment line below if user has been created
+-- from Cloud Dashboard:
+-- ALTER USER :username NOCREATEDB NOCREATEROLE;
+GRANT readwrite TO :username;

db/src/main/resources/sql/user/delete_user.sql

@@ -14,9 +14,7 @@
 --
 -- Script to delete a user from the database.
 
-REVOKE ALL PRIVILEGES ON DATABASE postgres FROM :username;
-REVOKE ALL PRIVILEGES ON SCHEMA public FROM :username;
-REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM :username;
-REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM :username;
-REVOKE ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public FROM :username;
+-- Ignore warnings like :username is not a member of role readonly/write.
+REVOKE readonly FROM :username;
+REVOKE readwrite FROM :username;
 DROP USER :username;

db/src/main/resources/sql/user/initialize_roles.sql

@@ -0,0 +1,48 @@
+-- Copyright 2019 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Initializes roles and their privileges in the postgres database in Cloud SQL.
+-- This script should run once under the **'postgres'** user before any other
+-- roles or users are created.
+
+-- Prevent backdoor grants through the implicit 'public' role.
+REVOKE ALL PRIVILEGES ON SCHEMA public from public;
+
+CREATE ROLE readonly;
+GRANT CONNECT ON DATABASE postgres TO readonly;
+GRANT USAGE ON SCHEMA public TO readonly;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO readonly;
+ALTER DEFAULT PRIVILEGES
+  IN SCHEMA public
+  FOR USER postgres
+  GRANT USAGE, SELECT ON SEQUENCES TO readonly;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO readonly;
+ALTER DEFAULT PRIVILEGES
+  IN SCHEMA public
+  FOR USER postgres
+  GRANT SELECT ON TABLES TO readonly;
+
+CREATE ROLE readwrite;
+GRANT CONNECT ON DATABASE postgres TO readwrite;
+GRANT USAGE ON SCHEMA public TO readwrite;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO readwrite;
+ALTER DEFAULT PRIVILEGES
+  IN SCHEMA public
+  FOR USER postgres
+  GRANT USAGE, SELECT ON SEQUENCES TO readwrite;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO readwrite;
+ALTER DEFAULT PRIVILEGES
+  IN SCHEMA public
+  FOR USER postgres
+  GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO readwrite;

db/src/main/resources/sql/user/set_flyway_privileges.sql

@@ -12,11 +12,9 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
--- Script to create a user with read-write permission to schema 'public' and
--- all tables.
+-- Removes write privileges to Flyway admin table from roles.
+-- This script is run once under 'postgres' after initialize_roles.sql
+-- has been run AND the initial schema deployment by Flyway is done.
 
-CREATE USER :username ENCRYPTED PASSWORD :'password';
-GRANT CONNECT ON DATABASE postgres TO :username;
-GRANT ALL PRIVILEGES ON SCHEMA public TO :username;
-GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :username;
-GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :username;
+REVOKE INSERT, UPDATE, DELETE ON TABLE public.flyway_schema_history FROM readonly;
+REVOKE INSERT, UPDATE, DELETE ON TABLE public.flyway_schema_history FROM readwrite;

gradle.properties

@@ -8,6 +8,9 @@ verboseTestOutput=false
 flowDocsFile=
 enableDependencyLocking=true
 
+# Comma separated list of test patterns, if specified run only these.
+testFilter=
+
 # GAE Environment for deployment and staging.
 environment=