Remove November 1 check (#859)

* Refactor ResaveEntitiesWithUniqueIdCommand to extract common methods

* Add a command to dedupe BillingEvent.Recurring entities

build.gradle

@@ -191,7 +191,7 @@ allprojects {
 }
 
 task runPresubmits(type: Exec) {
-  executable '/usr/bin/python'
+  executable '/usr/bin/python3'
   args('config/presubmits.py')
 }
 
@@ -260,7 +260,7 @@ subprojects {
       // in the 'configurations' block, the following code must run after
       // project evaluation, when all configurations have been created.
       configurations.each {
-        if (it.name != 'dependencyLicenseReport') {
+        if (it.name != 'dependencyLicenseReport' && it.name != 'integration') {
           it.resolutionStrategy.activateDependencyLocking()
         }
       }

buildSrc/gradle/dependency-locks/annotationProcessor.lockfile

@@ -1,25 +1,28 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto.value:auto-value:1.6.3
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

buildSrc/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/gradle/dependency-locks/annotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/gradle/dependency-locks/errorprone.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/gradle/dependency-locks/testingAnnotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/src/main/java/google/registry/util/DateTimeUtils.java

@@ -24,6 +24,7 @@ import java.time.ZonedDateTime;
 import java.util.TimeZone;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.LocalDate;
 
 /** Utilities methods and constants related to Joda {@link DateTime} objects. */
 public class DateTimeUtils {
@@ -108,4 +109,12 @@ public class DateTimeUtils {
         zonedDateTime.toInstant().toEpochMilli(),
         DateTimeZone.forTimeZone(TimeZone.getTimeZone(zonedDateTime.getZone())));
   }
+
+  public static java.sql.Date toSqlDate(LocalDate localDate) {
+    return new java.sql.Date(localDate.toDateTimeAtStartOfDay().getMillis());
+  }
+
+  public static LocalDate toLocalDate(java.sql.Date date) {
+    return new LocalDate(date.getTime(), DateTimeZone.UTC);
+  }
 }

common/src/main/java/google/registry/util/SystemSleeper.java

@@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.util.concurrent.Uninterruptibles;
 import java.io.Serializable;
-import java.util.concurrent.TimeUnit;
 import javax.annotation.concurrent.ThreadSafe;
 import javax.inject.Inject;
 import org.joda.time.ReadableDuration;
@@ -41,6 +40,6 @@ public final class SystemSleeper implements Sleeper, Serializable {
   @Override
   public void sleepUninterruptibly(ReadableDuration duration) {
     checkArgument(duration.getMillis() >= 0);
-    Uninterruptibles.sleepUninterruptibly(duration.getMillis(), TimeUnit.MILLISECONDS);
+    Uninterruptibles.sleepUninterruptibly(java.time.Duration.ofMillis(duration.getMillis()));
   }
 }

config/nom_build.py

@@ -16,7 +16,7 @@
 """
 
 import argparse
-import attr
+import dataclasses
 import io
 import os
 import shutil
@@ -25,7 +25,7 @@ import sys
 from typing import List, Union
 
 
-@attr.s(auto_attribs=True)
+@dataclasses.dataclass
 class Property:
     name : str = ''
     desc : str = ''
@@ -39,7 +39,7 @@ class Property:
                 raise ValidationError('value of {self.name} must be "true" or '
                                       '"false".')
 
-@attr.s(auto_attribs=True)
+@dataclasses.dataclass
 class GradleFlag:
     flags : Union[str, List[str]]
     desc : str
@@ -75,6 +75,7 @@ Pseudo-commands:
 """
 
 # Define all of our special gradle properties here.
+# TODO(b/169318491): use consistent naming style for properties and variables.
 PROPERTIES = [
     Property('mavenUrl',
              'URL to use for the main maven repository (defaults to maven '
@@ -124,6 +125,9 @@ PROPERTIES = [
              'server/schema integration tests. Please refer to <a '
              'href="./integration/README.md">integration project</a> for more '
              'information.'),
+    Property('baseSchemaTag',
+             'The nomulus version tag of the schema for use in the schema'
+             'deployment integration test (:db:schemaIncrementalDeployTest)'),
     Property('schema_version',
              'The nomulus version tag of the schema for use in a database'
              'integration test.'),

config/presubmits.py

@@ -18,6 +18,7 @@ Error Prone) so we must write them manually.
 """
 
 import os
+from typing import List, Tuple
 import sys
 import re
 
@@ -178,6 +179,90 @@ PRESUBMITS = {
         "JavaScript files should not include console logging."
 }
 
+# Note that this regex only works for one kind of Flyway file.  If we want to
+# start using "R" and "U" files we'll need to update this script.
+FLYWAY_FILE_RX = re.compile(r'V(\d+)__.*')
+
+
+def get_seqnum(filename: str, location: str) -> int:
+  """Extracts the sequence number from a filename."""
+  m = FLYWAY_FILE_RX.match(filename)
+  if m is None:
+    raise ValueError('Illegal Flyway filename: %s in %s' % (filename, location))
+  return int(m.group(1))
+
+
+def files_by_seqnum(files: List[str], location: str) -> List[Tuple[int, str]]:
+  """Returns the list of seqnum, filename sorted by sequence number."""
+  return [(get_seqnum(filename, location), filename) for filename in files]
+
+
+def has_valid_order(indexed_files: List[Tuple[int, str]], location: str) -> bool:
+  """Verify that sequence numbers are in order without gaps or duplicates.
+
+  Args:
+    files: List of seqnum, filename for a list of Flyway files.
+    location: Where the list of files came from (for error reporting).
+
+  Returns:
+    True if the file list is valid.
+  """
+  last_index = 0
+  valid = True
+  for seqnum, filename in indexed_files:
+    if seqnum == last_index:
+      print('duplicate Flyway file sequence number found in %s: %s' %
+            (location, filename))
+      valid = False
+    elif seqnum < last_index:
+      print('File %s in %s is out of order.' % (filename, location))
+      valid = False
+    elif seqnum != last_index + 1:
+      print('Missing Flyway sequence number %d in %s.  Next file is %s' %
+            (last_index + 1, location, filename))
+      valid = False
+    last_index = seqnum
+  return valid
+
+
+def verify_flyway_index():
+  """Verifies that the Flyway index file is in sync with the directory."""
+  success = True
+
+  # Sort the files in the Flyway directory by their sequence number.
+  files = sorted(
+      files_by_seqnum(os.listdir('db/src/main/resources/sql/flyway'),
+                      'Flyway directory'))
+
+  # Make sure that there are no gaps and no duplicate sequence numbers in the
+  # files themselves.
+  if not has_valid_order(files, 'Flyway directory'):
+    success = False
+
+  # Remove the sequence numbers and compare against the index file contents.
+  files = [filename[1] for filename in sorted(files)]
+  with open('db/src/main/resources/sql/flyway.txt') as index:
+    indexed_files = index.read().splitlines()
+  if files != indexed_files:
+    unindexed = set(files) - set(indexed_files)
+    if unindexed:
+      print('The following Flyway files are not in flyway.txt: %s' % unindexed)
+
+    nonexistent = set(indexed_files) - set(files)
+    if nonexistent:
+      print('The following files are in flyway.txt but not in the Flyway '
+            'directory: %s' % nonexistent)
+
+    # Do an ordering check on the index file (ignore the result, we're failing
+    # anyway).
+    has_valid_order(files_by_seqnum(indexed_files, 'flyway.txt'), 'flyway.txt')
+    success = False
+
+  if not success:
+    print('Please fix any conflicts and run "./nom_build :db:generateFlywayIndex"')
+
+  return not success
+
 
 def get_files():
   for root, dirnames, filenames in os.walk("."):
@@ -197,5 +282,10 @@ if __name__ == "__main__":
       failed = True
       print("%s had errors: \n  %s" % (file, "\n  ".join(error_messages)))
 
+  # And now for something completely different: check to see if the Flyway
+  # index is up-to-date.  It's quicker to do it here than in the unit tests:
+  # when we put it here it fails fast before all of the tests are run.
+  failed |= verify_flyway_index()
+
   if failed:
     sys.exit(1)

core/build.gradle

@@ -221,6 +221,7 @@ dependencies {
   compile deps['com.jcraft:jsch']
   testCompile deps['com.thoughtworks.qdox:qdox']
   compile deps['dnsjava:dnsjava']
+  runtime deps['guru.nidi:graphviz-java-all-j2v8']
   testCompile deps['io.github.classgraph:classgraph']
   testRuntime deps['io.github.java-diff-utils:java-diff-utils']
   testCompile deps['javax.annotation:javax.annotation-api']
@@ -245,6 +246,7 @@ dependencies {
   testCompile deps['org.apache.ftpserver:ftpserver-core']
   compile deps['org.apache.httpcomponents:httpclient']
   compile deps['org.apache.httpcomponents:httpcore']
+  runtime deps['org.apache.logging.log4j:log4j-core']
   testCompile deps['org.apache.sshd:sshd-core']
   testCompile deps['org.apache.sshd:sshd-scp']
   testCompile deps['org.apache.sshd:sshd-sftp']
@@ -257,6 +259,7 @@ dependencies {
   compile deps['org.hibernate:hibernate-core']
   compile deps['org.joda:joda-money']
   compile deps['org.json:json']
+  compile deps['org.jsoup:jsoup']
   testCompile deps['org.mortbay.jetty:jetty']
   compile deps['org.postgresql:postgresql']
   testCompile deps['org.seleniumhq.selenium:selenium-api']
@@ -269,6 +272,10 @@ dependencies {
   compile deps['org.testcontainers:postgresql']
   testCompile deps['org.testcontainers:selenium']
   testCompile deps['org.testcontainers:testcontainers']
+  compile deps['us.fatehi:schemacrawler']
+  compile deps['us.fatehi:schemacrawler-api']
+  compile deps['us.fatehi:schemacrawler-diagram']
+  compile deps['us.fatehi:schemacrawler-tools']
   compile deps['xerces:xmlParserAPIs']
   compile deps['xpp3:xpp3']
   // This dependency must come after javax.mail:mail as it would otherwise

core/gradle/dependency-locks/annotationProcessor.lockfile

@@ -1,8 +1,8 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto.value:auto-value:1.6.3
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
@@ -11,11 +11,11 @@ com.google.dagger:dagger-compiler:2.28
 com.google.dagger:dagger-producers:2.28
 com.google.dagger:dagger-spi:2.28
 com.google.dagger:dagger:2.28
-com.google.errorprone:error_prone_annotation:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
 com.google.errorprone:error_prone_annotations:2.3.4
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
@@ -30,11 +30,14 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:2.11.1
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.3.61
 org.jetbrains.kotlin:kotlin-stdlib:1.3.61
 org.jetbrains.kotlinx:kotlinx-metadata-jvm:0.1.0
 org.jetbrains:annotations:13.0
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

core/gradle/dependency-locks/compile.lockfile

@@ -202,6 +202,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +226,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -249,5 +251,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -200,6 +200,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -222,6 +223,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -246,5 +248,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/default.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -177,6 +184,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -198,11 +206,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -226,6 +237,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -240,6 +252,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -249,7 +263,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +235,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -238,6 +250,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/errorprone.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

core/gradle/dependency-locks/nonprodAnnotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -202,6 +202,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +226,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -249,5 +251,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -201,6 +201,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +225,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -248,5 +250,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/runtime.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +235,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -238,6 +250,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -1,8 +1,8 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto.value:auto-value:1.6.3
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
@@ -11,11 +11,11 @@ com.google.dagger:dagger-compiler:2.28
 com.google.dagger:dagger-producers:2.28
 com.google.dagger:dagger-spi:2.28
 com.google.dagger:dagger:2.28
-com.google.errorprone:error_prone_annotation:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
 com.google.errorprone:error_prone_annotations:2.3.4
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
@@ -30,11 +30,14 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:2.11.1
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.3.61
 org.jetbrains.kotlin:kotlin-stdlib:1.3.61
 org.jetbrains.kotlinx:kotlinx-metadata-jvm:0.1.0
 org.jetbrains:annotations:13.0
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

core/gradle/dependency-locks/testCompile.lockfile

@@ -248,6 +248,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -298,5 +299,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -247,6 +247,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -297,5 +298,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testRuntime.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.11.2
 com.fasterxml.jackson.core:jackson-core:2.11.2
 com.fasterxml.jackson.core:jackson-databind:2.11.2
@@ -138,6 +142,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.github.java-diff-utils:java-diff-utils:4.0
@@ -186,6 +193,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.13
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
@@ -216,7 +224,8 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -251,6 +260,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -289,6 +299,8 @@ org.seleniumhq.selenium:selenium-opera-driver:3.141.59
 org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -299,7 +311,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.11.2
 com.fasterxml.jackson.core:jackson-core:2.11.2
 com.fasterxml.jackson.core:jackson-databind:2.11.2
@@ -138,6 +142,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.github.java-diff-utils:java-diff-utils:4.0
@@ -186,6 +193,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.13
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
@@ -216,7 +224,8 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -251,6 +260,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -289,6 +299,8 @@ org.seleniumhq.selenium:selenium-opera-driver:3.141.59
 org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -300,7 +312,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -464,7 +464,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
         HostResource host = (HostResource) existingResource;
         if (host.isSubordinate()) {
           dnsQueue.addHostRefreshTask(host.getHostName());
-          tm().saveNewOrUpdate(
+          tm().put(
                   tm().load(host.getSuperordinateDomain())
                       .asBuilder()
                       .removeSubordinateHost(host.getHostName())

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -437,7 +437,7 @@ public final class Transforms {
                 .map(Optional::get)
                 .map(ofy::toPojo)
                 .collect(ImmutableList.toImmutableList());
-        retry(() -> jpaTm().transact(() -> jpaTm().saveNewOrUpdateAll(ofyEntities)));
+        retry(() -> jpaTm().transact(() -> jpaTm().putAll(ofyEntities)));
       }
     }
 

core/src/main/java/google/registry/beam/invoicing/sql/billing_events.sql

@@ -55,7 +55,7 @@ FROM (
     FROM
       `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
     WHERE
-      enableInvoicing IS TRUE) ) AS BillingEvent
+      invoicingEnabled IS TRUE) ) AS BillingEvent
   -- Gather billing ID from registrar table
   -- This is a 'JOIN' as opposed to 'LEFT JOIN' to filter out
   -- non-billable registrars

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -215,7 +215,7 @@ public class Spec11Pipeline implements Serializable {
                           .setRegistrarId(subdomain.registrarId())
                           .build();
                   JpaTransactionManager jpaTransactionManager = jpaSupplierFactory.get();
-                  jpaTransactionManager.transact(() -> jpaTransactionManager.saveNew(threatMatch));
+                  jpaTransactionManager.transact(() -> jpaTransactionManager.insert(threatMatch));
                 }
               }
             }));

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -16,9 +16,12 @@ package google.registry.config;
 
 import static com.google.common.base.Suppliers.memoize;
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap;
 import static google.registry.config.ConfigUtils.makeUrl;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.ResourceUtils.readResourceUtf8;
 import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import static java.util.Comparator.naturalOrder;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Ascii;
@@ -27,6 +30,7 @@ import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.util.TaskQueueUtils;
@@ -46,6 +50,7 @@ import javax.inject.Qualifier;
 import javax.inject.Singleton;
 import javax.mail.internet.AddressException;
 import javax.mail.internet.InternetAddress;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeConstants;
 import org.joda.time.Duration;
 
@@ -1345,6 +1350,33 @@ public final class RegistryConfig {
     public static String provideRdapTosStaticUrl(RegistryConfigSettings config) {
       return config.registryPolicy.rdapTosStaticUrl;
     }
+
+    @Provides
+    @Config("maxValidityDaysSchedule")
+    public static ImmutableSortedMap<DateTime, Integer> provideValidityDaysMap(
+        RegistryConfigSettings config) {
+      return config.sslCertificateValidation.maxValidityDaysSchedule.entrySet().stream()
+          .collect(
+              toImmutableSortedMap(
+                  naturalOrder(),
+                  e ->
+                      e.getKey().equals("START_OF_TIME")
+                          ? START_OF_TIME
+                          : DateTime.parse(e.getKey()),
+                  e -> e.getValue()));
+    }
+
+    @Provides
+    @Config("expirationWarningDays")
+    public static int provideDaysToExpiration(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.expirationWarningDays;
+    }
+
+    @Provides
+    @Config("minimumRsaKeyLength")
+    public static int provideMinimumRsaKeyLength(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.minimumRsaKeyLength;
+    }
   }
 
   /** Returns the App Engine project ID, which is based off the environment name. */

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -15,6 +15,7 @@
 package google.registry.config;
 
 import java.util.List;
+import java.util.Map;
 
 /** The POJO that YAML config files are deserialized into. */
 public class RegistryConfigSettings {
@@ -38,6 +39,7 @@ public class RegistryConfigSettings {
   public Beam beam;
   public Keyring keyring;
   public RegistryTool registryTool;
+  public SslCertificateValidation sslCertificateValidation;
 
   /** Configuration options that apply to the entire App Engine project. */
   public static class AppEngine {
@@ -218,4 +220,11 @@ public class RegistryConfigSettings {
     public String clientSecret;
     public String username;
   }
+
+  /** Configuration for the certificate checker. */
+  public static class SslCertificateValidation {
+    public Map<String, Integer> maxValidityDaysSchedule;
+    public int expirationWarningDays;
+    public int minimumRsaKeyLength;
+  }
 }

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -446,3 +446,17 @@ registryTool:
   # OAuth client secret used by the tool.
   clientSecret: YOUR_CLIENT_SECRET
   username: toolusername
+
+# Configuration options for checking SSL certificates.
+sslCertificateValidation:
+  # A map specifying the maximum amount of days the certificate can be valid.
+  # The entry key is the date closest before the date the certificate was issued
+  # and the entry value is the applicable maximum validity days for that certificate.
+  maxValidityDaysSchedule:
+    "START_OF_TIME": 825
+    "2020-09-01T00:00:00Z": 398
+  # The number of days before a certificate expires that indicates the
+  # certificate is nearing expiration and warnings should be sent.
+  expirationWarningDays: 30
+  # The minimum number of bits an RSA key must contain
+  minimumRsaKeyLength: 2048

core/src/main/java/google/registry/flows/certs/CertificateChecker.java

@@ -0,0 +1,221 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.flows.certs;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.Clock;
+import google.registry.util.DateTimeUtils;
+import java.io.ByteArrayInputStream;
+import java.security.PublicKey;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.security.interfaces.RSAPublicKey;
+import java.util.Date;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.Days;
+
+/** An utility to check that a given certificate meets our requirements */
+public class CertificateChecker {
+
+  private final ImmutableSortedMap<DateTime, Integer> maxValidityLengthSchedule;
+  private final int daysToExpiration;
+  private final int minimumRsaKeyLength;
+  private final Clock clock;
+
+  /**
+   * Constructs a CertificateChecker instance with the specified configuration parameters.
+   *
+   * <p>The max validity length schedule is a sorted map of {@link DateTime} to {@link Integer}
+   * entries representing a maximum validity period for certificates issued on or after that date.
+   * The first entry must have a key of {@link DateTimeUtils#START_OF_TIME}, such that every
+   * possible date has an applicable max validity period. Since security requirements tighten over
+   * time, the max validity periods will be decreasing as the date increases.
+   *
+   * <p>The validity length schedule used by all major Web browsers as of 2020Q4 would be
+   * represented as:
+   *
+   * <pre>
+   *   ImmutableSortedMap.of(
+   *     START_OF_TIME, 825,
+   *     DateTime.parse("2020-09-01T00:00:00Z"), 398
+   *   );
+   * </pre>
+   */
+  @Inject
+  public CertificateChecker(
+      @Config("maxValidityDaysSchedule")
+          ImmutableSortedMap<DateTime, Integer> maxValidityLengthSchedule,
+      @Config("expirationWarningDays") int daysToExpiration,
+      @Config("minimumRsaKeyLength") int minimumRsaKeyLength,
+      Clock clock) {
+    checkArgument(
+        maxValidityLengthSchedule.containsKey(START_OF_TIME),
+        "Max validity length schedule must contain an entry for START_OF_TIME");
+    this.maxValidityLengthSchedule = maxValidityLengthSchedule;
+    this.daysToExpiration = daysToExpiration;
+    this.minimumRsaKeyLength = minimumRsaKeyLength;
+    this.clock = clock;
+  }
+
+  /**
+   * Checks the given certificate string for violations and throws an exception if any violations
+   * exist.
+   */
+  public void validateCertificate(String certificateString) {
+    ImmutableSet<CertificateViolation> violations = checkCertificate(certificateString);
+    if (!violations.isEmpty()) {
+      String displayMessages =
+          violations.stream()
+              .map(violation -> getViolationDisplayMessage(violation))
+              .collect(Collectors.joining("\n"));
+      throw new IllegalArgumentException(displayMessages);
+    }
+  }
+
+  /**
+   * Checks a given certificate for violations and returns a list of all the violations the
+   * certificate has.
+   */
+  public ImmutableSet<CertificateViolation> checkCertificate(X509Certificate certificate) {
+    ImmutableSet.Builder<CertificateViolation> violations = new ImmutableSet.Builder<>();
+
+    // Check if currently in validity period
+    Date now = clock.nowUtc().toDate();
+    if (certificate.getNotAfter().before(now)) {
+      violations.add(CertificateViolation.EXPIRED);
+    } else if (certificate.getNotBefore().after(now)) {
+      violations.add(CertificateViolation.NOT_YET_VALID);
+    }
+
+    // Check validity period length
+    int maxValidityDays =
+        maxValidityLengthSchedule.floorEntry(new DateTime(certificate.getNotBefore())).getValue();
+    if (getValidityLengthInDays(certificate) > maxValidityDays) {
+      violations.add(CertificateViolation.VALIDITY_LENGTH_TOO_LONG);
+    }
+
+    // Check key strengths
+    PublicKey key = certificate.getPublicKey();
+    if (key.getAlgorithm().equals("RSA")) {
+      RSAPublicKey rsaPublicKey = (RSAPublicKey) key;
+      if (rsaPublicKey.getModulus().bitLength() < minimumRsaKeyLength) {
+        violations.add(CertificateViolation.RSA_KEY_LENGTH_TOO_SHORT);
+      }
+    } else if (key.getAlgorithm().equals("EC")) {
+      // TODO(sarahbot): Add verification of ECDSA curves
+    } else {
+      violations.add(CertificateViolation.ALGORITHM_CONSTRAINED);
+    }
+    return violations.build();
+  }
+
+  /**
+   * Converts a given string to a certificate and checks it for violations, returning a list of all
+   * the violations the certificate has.
+   */
+  public ImmutableSet<CertificateViolation> checkCertificate(String certificateString) {
+    X509Certificate certificate;
+
+    try {
+      certificate =
+          (X509Certificate)
+              CertificateFactory.getInstance("X509")
+                  .generateCertificate(new ByteArrayInputStream(certificateString.getBytes(UTF_8)));
+    } catch (CertificateException e) {
+      throw new IllegalArgumentException("Unable to read given certificate.");
+    }
+
+    return checkCertificate(certificate);
+  }
+
+  /**
+   * Returns whether the certificate is nearing expiration.
+   *
+   * <p>Note that this is <i>all</i> that it checks. The certificate itself may well be expired or
+   * not yet valid and this message will still return false. So you definitely want to pair a call
+   * to this method with a call to {@link #checkCertificate} to determine other issues with the
+   * certificate that may be occurring.
+   */
+  public boolean isNearingExpiration(X509Certificate certificate) {
+    Date nearingExpirationDate =
+        DateTime.parse(certificate.getNotAfter().toInstant().toString())
+            .minusDays(daysToExpiration)
+            .toDate();
+    return clock.nowUtc().toDate().after(nearingExpirationDate);
+  }
+
+  private static int getValidityLengthInDays(X509Certificate certificate) {
+    DateTime start = DateTime.parse(certificate.getNotBefore().toInstant().toString());
+    DateTime end = DateTime.parse(certificate.getNotAfter().toInstant().toString());
+    return Days.daysBetween(start.withTimeAtStartOfDay(), end.withTimeAtStartOfDay()).getDays();
+  }
+
+  private String getViolationDisplayMessage(CertificateViolation certificateViolation) {
+    // Yes, we'd rather do this as an instance method on the CertificateViolation enum itself, but
+    // we can't because we need access to configuration (injected as instance variables) which you
+    // can't get in a static enum context.
+    switch (certificateViolation) {
+      case EXPIRED:
+        return "Certificate is expired.";
+      case NOT_YET_VALID:
+        return "Certificate start date is in the future.";
+      case ALGORITHM_CONSTRAINED:
+        return "Certificate key algorithm must be RSA or ECDSA.";
+      case RSA_KEY_LENGTH_TOO_SHORT:
+        return String.format(
+            "RSA key length is too short; the minimum allowed length is %d bits.",
+            this.minimumRsaKeyLength);
+      case VALIDITY_LENGTH_TOO_LONG:
+        return String.format(
+            "Certificate validity period is too long; it must be less than or equal to %d days.",
+            this.maxValidityLengthSchedule.lastEntry().getValue());
+      default:
+        throw new IllegalArgumentException(
+            String.format(
+                "Unknown CertificateViolation enum value: %s", certificateViolation.name()));
+    }
+  }
+
+  /**
+   * The type of violation a certificate has based on the certificate requirements
+   * (go/registry-proxy-security).
+   */
+  public enum CertificateViolation {
+    EXPIRED,
+    NOT_YET_VALID,
+    VALIDITY_LENGTH_TOO_LONG,
+    RSA_KEY_LENGTH_TOO_SHORT,
+    ALGORITHM_CONSTRAINED;
+
+    /**
+     * Gets a suitable end-user-facing display message for this particular certificate violation.
+     *
+     * <p>Note that the {@link CertificateChecker} instance must be passed in because it contains
+     * configuration values (e.g. minimum RSA key length) that go into the error message text.
+     */
+    public String getDisplayMessage(CertificateChecker certificateChecker) {
+      return certificateChecker.getViolationDisplayMessage(this);
+    }
+  }
+}

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -173,7 +173,7 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
             existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId);
     updateForeignKeyIndexDeletionTime(newDomain);
     entitiesToSave.add(newDomain, historyEntry, autorenewEvent, autorenewPollMessage);
-    tm().saveNewOrUpdateAll(entitiesToSave.build());
+    tm().putAll(entitiesToSave.build());
     tm().delete(existingDomain.getDeletePollMessage());
     dnsQueue.addDomainRefreshTask(existingDomain.getDomainName());
     return responseBuilder

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -67,6 +67,7 @@ import google.registry.model.domain.DomainCommand.Update.AddRemove;
 import google.registry.model.domain.DomainCommand.Update.Change;
 import google.registry.model.domain.fee.FeeUpdateCommandExtension;
 import google.registry.model.domain.metadata.MetadataExtension;
+import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.domain.secdns.SecDnsUpdateExtension;
 import google.registry.model.domain.superuser.DomainUpdateSuperuserExtension;
 import google.registry.model.eppcommon.AuthInfo;
@@ -238,10 +239,16 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     DomainBase.Builder domainBuilder =
         domain
             .asBuilder()
-            // Handle the secDNS extension.
+            // Handle the secDNS extension. As dsData in secDnsUpdate is read from EPP input and
+            // does not have domainRepoId set, we create a copy of the existing dsData without
+            // domainRepoId for comparison.
             .setDsData(
                 secDnsUpdate.isPresent()
-                    ? updateDsData(domain.getDsData(), secDnsUpdate.get())
+                    ? updateDsData(
+                        domain.getDsData().stream()
+                            .map(DelegationSignerData::cloneWithoutDomainRepoId)
+                            .collect(toImmutableSet()),
+                        secDnsUpdate.get())
                     : domain.getDsData())
             .setLastEppUpdateTime(now)
             .setLastEppUpdateClientId(clientId)

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -285,7 +285,7 @@ public final class HostUpdateFlow implements TransactionalFlow {
         && newHost.isSubordinate()
         && Objects.equals(
             existingHost.getSuperordinateDomain(), newHost.getSuperordinateDomain())) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(existingHost.getSuperordinateDomain())
                   .asBuilder()
                   .removeSubordinateHost(existingHost.getHostName())
@@ -294,14 +294,14 @@ public final class HostUpdateFlow implements TransactionalFlow {
       return;
     }
     if (existingHost.isSubordinate()) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(existingHost.getSuperordinateDomain())
                   .asBuilder()
                   .removeSubordinateHost(existingHost.getHostName())
                   .build());
     }
     if (newHost.isSubordinate()) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(newHost.getSuperordinateDomain())
                   .asBuilder()
                   .addSubordinateHost(newHost.getHostName())

core/src/main/java/google/registry/flows/poll/PollFlowUtils.java

@@ -64,7 +64,7 @@ public final class PollFlowUtils {
       // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
       // autorenew poll message has no more events to deliver and should be deleted.
       if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
-        tm().saveNewOrUpdate(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
+        tm().put(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
         includeAckedMessageInCount = isBeforeOrAt(nextEventTime, tm().getTransactionTime());
       } else {
         tm().delete(autorenewPollMessage.createVKey());

core/src/main/java/google/registry/idn/Chinese-IDN.txt

@@ -1,5 +1,5 @@
 # Registry: Charleston Road Registry Inc.
-# Language: Chinese
+# Script: zh-Hans
 # Version: 1.0
 # Effective Date: 04-12-2012
 #

core/src/main/java/google/registry/idn/Traditional-Chinese-IDN.txt

@@ -1,5 +1,5 @@
 # Registry: Charleston Road Registry Inc.
-# Language: Traditional Chinese
+# Script: zh-Hant
 # Version: 1.0
 # Effective Date: 04-12-2012
 # Contact: iana-contact@google.com

core/src/main/java/google/registry/model/EppResource.java

@@ -24,7 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
@@ -62,23 +61,32 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   /**
    * Unique identifier in the registry for this resource.
    *
+   * <p>Not persisted so that we can store these in references to other objects. Subclasses that
+   * wish to use this as the primary key should create a getter method annotated with @Id
+   *
    * <p>This is in the (\w|_){1,80}-\w{1,8} format specified by RFC 5730 for roidType.
    *
    * @see <a href="https://tools.ietf.org/html/rfc5730">RFC 5730</a>
    */
-  @Id
-  // not persisted so that we can store these in references to other objects. Subclasses that wish
-  // to use this as the primary key should create a getter method annotated with @Id
-  @Transient
-  String repoId;
+  @Id @Transient String repoId;
 
-  /** The ID of the registrar that is currently sponsoring this resource. */
+  /**
+   * The ID of the registrar that is currently sponsoring this resource.
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
   @Index
-  @Column(name = "currentSponsorRegistrarId", nullable = false)
+  @Column(name = "currentSponsorRegistrarId")
   String currentSponsorClientId;
 
-  /** The ID of the registrar that created this resource. */
-  @Column(name = "creationRegistrarId", nullable = false)
+  /**
+   * The ID of the registrar that created this resource.
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
+  @Column(name = "creationRegistrarId")
   String creationClientId;
 
   /**
@@ -91,13 +99,17 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   @Column(name = "lastEppUpdateRegistrarId")
   String lastEppUpdateClientId;
 
-  /** The time when this resource was created. */
-  // Map the method to XML, not the field, because if we map the field (with an adaptor class) it
-  // will never be omitted from the xml even if the timestamp inside creationTime is null and we
-  // return null from the adaptor. (Instead it gets written as an empty tag.)
-  @Column(nullable = false)
-  @Index
-  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
+  /**
+   * The time when this resource was created.
+   *
+   * <p>Map the method to XML, not the field, because if we map the field (with an adaptor class) it
+   * will never be omitted from the xml even if the timestamp inside creationTime is null and we
+   * return null from the adaptor (instead it gets written as an empty tag).
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
+  @Index CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.
@@ -112,8 +124,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * out of the index at that time, as long as we query for resources whose deletion time is before
    * now.
    */
-  @Index
-  DateTime deletionTime;
+  @Index DateTime deletionTime;
 
   /**
    * The time that this resource was last updated.
@@ -144,7 +155,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     return repoId;
   }
 
-  // Hibernate needs this to populate the repo ID, but no one else should ever use it
+  /** This method exists solely to satisfy Hibernate. Use {@link Builder} instead. */
   @SuppressWarnings("UnusedMethod")
   private void setRepoId(String repoId) {
     this.repoId = repoId;
@@ -375,7 +386,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   private static LoadingCache<VKey<? extends EppResource>, EppResource> createEppResourcesCache(
       Duration expiry) {
     return CacheBuilder.newBuilder()
-        .expireAfterWrite(expiry.getMillis(), MILLISECONDS)
+        .expireAfterWrite(java.time.Duration.ofMillis(expiry.getMillis()))
         .maximumSize(getEppResourceMaxCachedEntries())
         .build(CACHE_LOADER);
   }

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -61,7 +61,17 @@ public abstract class ImmutableObject implements Cloneable {
   private boolean equalsImmutableObject(ImmutableObject other) {
     return getClass().equals(other.getClass())
         && hashCode() == other.hashCode()
-        && ModelUtils.getFieldValues(this).equals(ModelUtils.getFieldValues(other));
+        && getSignificantFields().equals(other.getSignificantFields());
+  }
+
+  /**
+   * Returns the map of significant fields (fields that we care about for purposes of comparison and
+   * display).
+   *
+   * <p>Isolated into a method so that derived classes can override it.
+   */
+  protected Map<Field, Object> getSignificantFields() {
+    return ModelUtils.getFieldValues(this);
   }
 
   @Override
@@ -72,7 +82,7 @@ public abstract class ImmutableObject implements Cloneable {
   @Override
   public int hashCode() {
     if (hashCode == null) {
-      hashCode = Arrays.hashCode(ModelUtils.getFieldValues(this).values().toArray());
+      hashCode = Arrays.hashCode(getSignificantFields().values().toArray());
     }
     return hashCode;
   }
@@ -111,7 +121,7 @@ public abstract class ImmutableObject implements Cloneable {
   @Override
   public String toString() {
     NavigableMap<String, Object> sortedFields = new TreeMap<>();
-    for (Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
+    for (Entry<Field, Object> entry : getSignificantFields().entrySet()) {
       sortedFields.put(entry.getKey().getName(), entry.getValue());
     }
     return toStringHelper(sortedFields);
@@ -121,7 +131,7 @@ public abstract class ImmutableObject implements Cloneable {
   public String toHydratedString() {
     // We can't use ImmutableSortedMap because we need to allow null values.
     NavigableMap<String, Object> sortedFields = new TreeMap<>();
-    for (Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
+    for (Entry<Field, Object> entry : getSignificantFields().entrySet()) {
       Field field = entry.getKey();
       Object value = entry.getValue();
       sortedFields.put(
@@ -161,7 +171,7 @@ public abstract class ImmutableObject implements Cloneable {
       // LinkedHashMap to preserve field ordering and because ImmutableMap forbids null
       // values.
       Map<String, Object> result = new LinkedHashMap<>();
-      for (Entry<Field, Object> entry : ModelUtils.getFieldValues(o).entrySet()) {
+      for (Entry<Field, Object> entry : ((ImmutableObject) o).getSignificantFields().entrySet()) {
         Field field = entry.getKey();
         if (!field.isAnnotationPresent(IgnoredInDiffableMap.class)) {
           result.put(field.getName(), toMapRecursive(entry.getValue()));

core/src/main/java/google/registry/model/ModelUtils.java

@@ -194,7 +194,7 @@ public class ModelUtils {
    * returned map in its implementation of {@link ImmutableObject#toString} and {@link
    * ImmutableObject#equals}, which work by comparing and printing these maps.
    */
-  static Map<Field, Object> getFieldValues(Object instance) {
+  public static Map<Field, Object> getFieldValues(Object instance) {
     // Don't make this ImmutableMap because field values can be null.
     Map<Field, Object> values = new LinkedHashMap<>();
     for (Field field : getAllFields(instance.getClass()).values()) {

core/src/main/java/google/registry/model/OteAccountBuilder.java

@@ -252,7 +252,8 @@ public final class OteAccountBuilder {
   private void saveAllEntities() {
     tm().assertInTransaction();
 
-    ImmutableList<Registry> registries = ImmutableList.of(sunriseTld, gaTld, eapTld);
+    // use ImmutableObject instead of Registry so that the Key generation doesn't break
+    ImmutableList<ImmutableObject> registries = ImmutableList.of(sunriseTld, gaTld, eapTld);
     ImmutableList<RegistrarContact> contacts = contactsBuilder.build();
 
     if (!replaceExisting) {

core/src/main/java/google/registry/model/OteStats.java

@@ -59,25 +59,11 @@ public class OteStats {
 
   private OteStats() {}
 
-  private static final Predicate<EppInput> HAS_CLAIMS_NOTICE =
-      eppInput -> {
-        Optional<LaunchCreateExtension> launchCreate =
-            eppInput.getSingleExtension(LaunchCreateExtension.class);
-        return launchCreate.isPresent() && launchCreate.get().getNotice() != null;
-      };
-
   private static final Predicate<EppInput> HAS_SEC_DNS =
       eppInput ->
           eppInput.getSingleExtension(SecDnsCreateExtension.class).isPresent()
               || eppInput.getSingleExtension(SecDnsUpdateExtension.class).isPresent();
 
-  private static final Predicate<EppInput> IS_SUNRISE =
-      eppInput -> {
-        Optional<LaunchCreateExtension> launchCreate =
-            eppInput.getSingleExtension(LaunchCreateExtension.class);
-        return launchCreate.isPresent() && !isNullOrEmpty(launchCreate.get().getSignedMarks());
-      };
-
   private static final Predicate<EppInput> IS_IDN =
       eppInput ->
           ((DomainCommand.Create)
@@ -94,6 +80,18 @@ public class OteStats {
                           .getResourceCommand())
                   .getInetAddresses());
 
+  private static boolean hasClaimsNotice(EppInput eppInput) {
+    Optional<LaunchCreateExtension> launchCreate =
+        eppInput.getSingleExtension(LaunchCreateExtension.class);
+    return launchCreate.isPresent() && launchCreate.get().getNotice() != null;
+  }
+
+  private static boolean isSunrise(EppInput eppInput) {
+    Optional<LaunchCreateExtension> launchCreate =
+        eppInput.getSingleExtension(LaunchCreateExtension.class);
+    return launchCreate.isPresent() && !isNullOrEmpty(launchCreate.get().getSignedMarks());
+  }
+
   /** Enum defining the distinct statistics (types of registrar actions) to record. */
   public enum StatType {
     CONTACT_CREATES(0, equalTo(Type.CONTACT_CREATE)),
@@ -107,8 +105,8 @@ public class OteStats {
     DOMAIN_CREATES(0, equalTo(Type.DOMAIN_CREATE)),
     DOMAIN_CREATES_ASCII(1, equalTo(Type.DOMAIN_CREATE), IS_IDN.negate()),
     DOMAIN_CREATES_IDN(1, equalTo(Type.DOMAIN_CREATE), IS_IDN),
-    DOMAIN_CREATES_START_DATE_SUNRISE(1, equalTo(Type.DOMAIN_CREATE), IS_SUNRISE),
-    DOMAIN_CREATES_WITH_CLAIMS_NOTICE(1, equalTo(Type.DOMAIN_CREATE), HAS_CLAIMS_NOTICE),
+    DOMAIN_CREATES_START_DATE_SUNRISE(1, equalTo(Type.DOMAIN_CREATE), OteStats::isSunrise),
+    DOMAIN_CREATES_WITH_CLAIMS_NOTICE(1, equalTo(Type.DOMAIN_CREATE), OteStats::hasClaimsNotice),
     DOMAIN_CREATES_WITH_FEE(
         1,
         equalTo(Type.DOMAIN_CREATE),

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -24,6 +24,7 @@ import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
@@ -33,12 +34,14 @@ import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.TimeOfYear;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.GracePeriod;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.token.AllocationToken;
@@ -47,6 +50,8 @@ import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -57,16 +62,14 @@ import javax.persistence.Column;
 import javax.persistence.Embedded;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.MappedSuperclass;
+import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
 /** A billable event in a domain's lifecycle. */
 @MappedSuperclass
-@WithLongVKey
 public abstract class BillingEvent extends ImmutableObject
     implements Buildable, TransferServerApproveEntity {
 
@@ -108,10 +111,7 @@ public abstract class BillingEvent extends ImmutableObject
   }
 
   /** Entity id. */
-  @Id
-  @javax.persistence.Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
-  Long id;
+  @Id @javax.persistence.Id Long id;
 
   @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
 
@@ -149,6 +149,21 @@ public abstract class BillingEvent extends ImmutableObject
   @Nullable
   Set<Flag> flags;
 
+  @PostLoad
+  void postLoad() {
+    parent =
+        Key.create(
+            Key.create(DomainBase.class, domainRepoId),
+            HistoryEntry.class,
+            domainHistoryRevisionId);
+  }
+
+  @OnLoad
+  void onLoad() {
+    domainHistoryRevisionId = parent.getId();
+    domainRepoId = parent.getParent().getName();
+  }
+
   public String getClientId() {
     return clientId;
   }
@@ -245,7 +260,6 @@ public abstract class BillingEvent extends ImmutableObject
     }
 
     public B setParent(Key<HistoryEntry> parentKey) {
-      // TODO(shicong): Figure out how to set domainHistoryRevisionId and domainRepoId
       getInstance().parent = parentKey;
       return thisCastToDerived();
     }
@@ -258,6 +272,11 @@ public abstract class BillingEvent extends ImmutableObject
       checkNotNull(instance.eventTime, "Event time must be set");
       checkNotNull(instance.targetId, "Target ID must be set");
       checkNotNull(instance.parent, "Parent must be set");
+      checkNotNull(instance.parent.getParent(), "parent.getParent() must be set");
+      checkNotNull(
+          instance.parent.getParent().getName(), "parent.getParent().getName() must be set");
+      instance.domainHistoryRevisionId = instance.parent.getId();
+      instance.domainRepoId = instance.parent.getParent().getName();
       return super.build();
     }
   }
@@ -275,6 +294,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "allocation_token_id")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
+  @WithLongVKey
   public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
@@ -310,7 +330,7 @@ public abstract class BillingEvent extends ImmutableObject
      * Cancellation}s.
      */
     @Column(name = "cancellation_matching_billing_recurrence_id")
-    VKey<? extends BillingEvent> cancellationMatchingBillingEvent;
+    VKey<Recurring> cancellationMatchingBillingEvent;
 
     /**
      * The {@link AllocationToken} used in the creation of this event, or null if one was not used.
@@ -392,7 +412,7 @@ public abstract class BillingEvent extends ImmutableObject
       }
 
       public Builder setCancellationMatchingBillingEvent(
-          VKey<? extends BillingEvent> cancellationMatchingBillingEvent) {
+          VKey<Recurring> cancellationMatchingBillingEvent) {
         getInstance().cancellationMatchingBillingEvent = cancellationMatchingBillingEvent;
         return this;
       }
@@ -451,6 +471,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
+  @WithLongVKey
   public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity {
 
     /**
@@ -545,6 +566,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "billingTime")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
+  @WithLongVKey
   public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billing time of the charge that is being cancelled. */
@@ -665,7 +687,8 @@ public abstract class BillingEvent extends ImmutableObject
   /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
-  public static class Modification extends BillingEvent implements DatastoreAndSqlEntity {
+  @WithLongVKey
+  public static class Modification extends BillingEvent implements DatastoreEntity {
 
     /** The change in cost that should be applied to the original billing event. */
     Money cost;
@@ -727,6 +750,11 @@ public abstract class BillingEvent extends ImmutableObject
           .build();
     }
 
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not persisted in SQL
+    }
+
     /** A builder for {@link Modification} since it is immutable. */
     public static class Builder extends BillingEvent.Builder<Modification, Builder> {
 

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -14,18 +14,25 @@
 
 package google.registry.model.contact;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
-import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.contact.ContactHistory.ContactHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
 
 /**
  * A persisted history entry representing an EPP modification to a contact.
@@ -44,16 +51,27 @@ import javax.persistence.Id;
     })
 @EntitySubclass
 @Access(AccessType.FIELD)
-public class ContactHistory extends HistoryEntry {
+@IdClass(ContactHistoryId.class)
+public class ContactHistory extends HistoryEntry implements SqlEntity {
 
   // Store ContactBase instead of ContactResource so we don't pick up its @Id
-  ContactBase contactBase;
-
-  @Column(nullable = false)
-  VKey<ContactResource> contactRepoId;
+  @Nullable ContactBase contactBase;
+
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getContactRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setContactRepoId(String contactRepoId) {
+    parent = Key.create(ContactResource.class, contactRepoId);
+  }
 
   @Id
-  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "TempHistorySequenceGenerator")
   @Column(name = "historyRevisionId")
   @Access(AccessType.PROPERTY)
   @Override
@@ -61,14 +79,98 @@ public class ContactHistory extends HistoryEntry {
     return super.getId();
   }
 
-  /** The state of the {@link ContactBase} object at this point in time. */
-  public ContactBase getContactBase() {
-    return contactBase;
+  /**
+   * The values of all the fields on the {@link ContactBase} object after the action represented by
+   * this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<ContactBase> getContactBase() {
+    return Optional.ofNullable(contactBase);
   }
 
   /** The key to the {@link ContactResource} this is based off of. */
-  public VKey<ContactResource> getContactRepoId() {
-    return contactRepoId;
+  public VKey<ContactResource> getParentVKey() {
+    return VKey.create(ContactResource.class, getContactRepoId());
+  }
+
+  /** Creates a {@link VKey} instance for this entity. */
+  public VKey<ContactHistory> createVKey() {
+    return VKey.create(
+        ContactHistory.class, new ContactHistoryId(getContactRepoId(), getId()), Key.create(this));
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Normally Hibernate would see that the contact fields are all null and would fill contactBase
+    // with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+    if (contactBase != null && contactBase.getContactId() == null) {
+      contactBase = null;
+    }
+  }
+
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
+  /** Class to represent the composite primary key of {@link ContactHistory} entity. */
+  static class ContactHistoryId extends ImmutableObject implements Serializable {
+
+    private String contactRepoId;
+
+    private Long id;
+
+    /** Hibernate requires this default constructor. */
+    private ContactHistoryId() {}
+
+    ContactHistoryId(String contactRepoId, long id) {
+      this.contactRepoId = contactRepoId;
+      this.id = id;
+    }
+
+    /**
+     * Returns the contact repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getContactRepoId() {
+      return contactRepoId;
+    }
+
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
+      return id;
+    }
+
+    /**
+     * Sets the contact repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setContactRepoId(String contactRepoId) {
+      this.contactRepoId = contactRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
+      this.id = id;
+    }
   }
 
   @Override
@@ -89,18 +191,8 @@ public class ContactHistory extends HistoryEntry {
       return this;
     }
 
-    public Builder setContactRepoId(VKey<ContactResource> contactRepoId) {
-      getInstance().contactRepoId = contactRepoId;
-      contactRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
-      return this;
-    }
-
-    // We can remove this once all HistoryEntries are converted to History objects
-    @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().contactRepoId =
-          VKey.create(ContactResource.class, parent.getName(), (Key<ContactResource>) parent);
+    public Builder setContactRepoId(String contactRepoId) {
+      getInstance().parent = Key.create(ContactResource.class, contactRepoId);
       return this;
     }
   }

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -51,7 +51,6 @@ public class ContactResource extends ContactBase
 
   @Override
   public VKey<ContactResource> createVKey() {
-    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.create(ContactResource.class, getRepoId(), Key.create(this));
   }
 

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -19,6 +19,7 @@ import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
@@ -106,6 +107,32 @@ public class DomainBase extends DomainContent
     return gracePeriods;
   }
 
+  /**
+   * Returns the set of {@link DelegationSignerData} associated with the domain.
+   *
+   * <p>This is the getter method specific for Hibernate to access the field so it is set to
+   * private. The caller can use the public {@link #getDsData()} to get the DS data.
+   *
+   * <p>Note that we need to set `insertable = false, updatable = false` for @JoinColumn, otherwise
+   * Hibernate would try to set the foreign key to null(through an UPDATE TABLE sql) instead of
+   * deleting the whole entry from the table when the {@link DelegationSignerData} is removed from
+   * the set.
+   */
+  @Access(AccessType.PROPERTY)
+  @OneToMany(
+      cascade = {CascadeType.ALL},
+      fetch = FetchType.EAGER,
+      orphanRemoval = true)
+  @JoinColumn(
+      name = "domainRepoId",
+      referencedColumnName = "repoId",
+      insertable = false,
+      updatable = false)
+  @SuppressWarnings("UnusedMethod")
+  private Set<DelegationSignerData> getInternalDelegationSignerData() {
+    return dsData;
+  }
+
   @Override
   public VKey<DomainBase> createVKey() {
     return VKey.create(DomainBase.class, getRepoId(), Key.create(this));

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -35,11 +35,13 @@ import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static google.registry.util.DomainNameUtils.getTldFromDomainName;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Ordering;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Streams;
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
@@ -49,6 +51,7 @@ import google.registry.flows.ResourceFlowUtils;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.launch.LaunchNotice;
 import google.registry.model.domain.rgp.GracePeriodStatus;
@@ -57,6 +60,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registry.Registry;
+import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
@@ -65,7 +69,6 @@ import java.util.HashSet;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.function.Predicate;
 import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -209,6 +212,15 @@ public class DomainContent extends EppResource
   @Column(name = "deletion_poll_message_id")
   VKey<PollMessage.OneTime> deletePollMessage;
 
+  /**
+   * History record for the delete poll message.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Column(name = "deletion_poll_message_history_id")
+  @Ignore
+  Long deletePollMessageHistoryId;
+
   /**
    * The recurring billing event associated with this domain's autorenewals.
    *
@@ -220,6 +232,15 @@ public class DomainContent extends EppResource
   @Column(name = "billing_recurrence_id")
   VKey<BillingEvent.Recurring> autorenewBillingEvent;
 
+  /**
+   * History record for the autorenew billing event.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Column(name = "billing_recurrence_history_id")
+  @Ignore
+  Long autorenewBillingEventHistoryId;
+
   /**
    * The recurring poll message associated with this domain's autorenewals.
    *
@@ -231,6 +252,13 @@ public class DomainContent extends EppResource
   @Column(name = "autorenew_poll_message_id")
   VKey<PollMessage.Autorenew> autorenewPollMessage;
 
+  /**
+   * History record for the autorenew poll message.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Ignore Long autorenewPollMessageHistoryId;
+
   /** The unexpired grace periods for this domain (some of which may not be active yet). */
   @Transient Set<GracePeriod> gracePeriods;
 
@@ -275,19 +303,28 @@ public class DomainContent extends EppResource
         allContacts.stream().map(DesignatedContact::reconstitute).collect(toImmutableSet());
     setContactFields(allContacts, true);
 
-    // We have to return the cloned object here because the original object's
-    // hashcode is not correct due to the change to its domainRepoId. The cloned
-    // object will have a null hashcode so that it can get a recalculated hashcode
-    // when its hashCode() is invoked.
+    // We have to return the cloned object here because the original object's hashcode is not
+    // correct due to the change to its domainRepoId and history ids. The cloned object will have a
+    // null hashcode so that it can get a recalculated hashcode when its hashCode() is invoked.
     // TODO(b/162739503): Remove this after fully migrating to Cloud SQL.
     gracePeriods =
         nullToEmptyImmutableCopy(gracePeriods).stream()
-            .map(gracePeriod -> gracePeriod.cloneWithDomainRepoId(getRepoId()))
+            .map(gracePeriod -> gracePeriod.cloneAfterOfyLoad(getRepoId()))
+            .collect(toImmutableSet());
+
+    // Restore history record ids.
+    autorenewPollMessageHistoryId = getHistoryId(autorenewPollMessage);
+    autorenewBillingEventHistoryId = getHistoryId(autorenewBillingEvent);
+    deletePollMessageHistoryId = getHistoryId(deletePollMessage);
+    dsData =
+        nullToEmptyImmutableCopy(dsData).stream()
+            .map(dsData -> dsData.cloneWithDomainRepoId(getRepoId()))
             .collect(toImmutableSet());
   }
 
   @PostLoad
-  void postLoad() {
+  @SuppressWarnings("UnusedMethod")
+  private final void postLoad() {
     // Reconstitute the contact list.
     ImmutableSet.Builder<DesignatedContact> contactsBuilder = new ImmutableSet.Builder<>();
 
@@ -306,6 +343,27 @@ public class DomainContent extends EppResource
     }
 
     allContacts = contactsBuilder.build();
+
+    // Reconstitute the composite ofy keys from the SQL data.
+    Key<DomainBase> myKey = Key.create(DomainBase.class, getRepoId());
+    deletePollMessage = restoreOfyFrom(myKey, deletePollMessage, deletePollMessageHistoryId);
+    autorenewBillingEvent =
+        restoreOfyFrom(myKey, autorenewBillingEvent, autorenewBillingEventHistoryId);
+    autorenewPollMessage =
+        restoreOfyFrom(myKey, autorenewPollMessage, autorenewPollMessageHistoryId);
+
+    if (transferData != null) {
+      transferData.restoreOfyKeys(myKey);
+    }
+  }
+
+  public static <T> VKey<T> restoreOfyFrom(Key<DomainBase> domainKey, VKey<T> key, Long historyId) {
+    if (historyId == null) {
+      // This is a legacy key (or a null key, in which case this works too)
+      return VKey.restoreOfyFrom(key, EntityGroupRoot.class, "per-tld");
+    } else {
+      return VKey.restoreOfyFrom(key, domainKey, HistoryEntry.class, historyId);
+    }
   }
 
   public ImmutableSet<String> getSubordinateHosts() {
@@ -387,6 +445,12 @@ public class DomainContent extends EppResource
     this.gracePeriods = gracePeriods;
   }
 
+  // Hibernate needs this in order to populate dsData but no one else should ever use it
+  @SuppressWarnings("UnusedMethod")
+  private void setInternalDelegationSignerData(Set<DelegationSignerData> dsData) {
+    this.dsData = dsData;
+  }
+
   public final String getCurrentSponsorClientId() {
     return getPersistedCurrentSponsorClientId();
   }
@@ -655,13 +719,31 @@ public class DomainContent extends EppResource
             + " use DomainBase instead");
   }
 
+  /**
+   * Obtains a history id from the given key.
+   *
+   * <p>The key must be a composite key either of the form domain-key/history-key/long-event-key or
+   * EntityGroupRoot/long-event-key (for legacy keys). In the latter case or for a null key returns
+   * a history id of null.
+   */
+  public static Long getHistoryId(VKey<?> key) {
+    if (key == null) {
+      return null;
+    }
+    Key<?> parent = key.getOfyKey().getParent();
+    if (parent == null || parent.getKind().equals("EntityGroupRoot")) {
+      return null;
+    }
+    return parent.getId();
+  }
+
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
   static final Predicate<DesignatedContact> IS_REGISTRANT =
       (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);
 
   /** An override of {@link EppResource#asBuilder} with tighter typing. */
   @Override
-  public Builder asBuilder() {
+  public Builder<? extends DomainContent, ?> asBuilder() {
     return new Builder<>(clone(this));
   }
 
@@ -698,10 +780,16 @@ public class DomainContent extends EppResource
       instance.tld = getTldFromDomainName(instance.fullyQualifiedDomainName);
 
       T newDomain = super.build();
-      // Hibernate throws exception if gracePeriods is null because we enabled all cascadable
-      // operations and orphan removal.
+      // Hibernate throws exception if gracePeriods or dsData is null because we enabled all
+      // cascadable operations and orphan removal.
       newDomain.gracePeriods =
           newDomain.gracePeriods == null ? ImmutableSet.of() : newDomain.gracePeriods;
+      newDomain.dsData =
+          newDomain.dsData == null
+              ? ImmutableSet.of()
+              : newDomain.dsData.stream()
+                  .map(ds -> ds.cloneWithDomainRepoId(instance.getRepoId()))
+                  .collect(toImmutableSet());
       return newDomain;
     }
 
@@ -824,16 +912,19 @@ public class DomainContent extends EppResource
 
     public B setDeletePollMessage(VKey<PollMessage.OneTime> deletePollMessage) {
       getInstance().deletePollMessage = deletePollMessage;
+      getInstance().deletePollMessageHistoryId = getHistoryId(deletePollMessage);
       return thisCastToDerived();
     }
 
     public B setAutorenewBillingEvent(VKey<BillingEvent.Recurring> autorenewBillingEvent) {
       getInstance().autorenewBillingEvent = autorenewBillingEvent;
+      getInstance().autorenewBillingEventHistoryId = getHistoryId(autorenewBillingEvent);
       return thisCastToDerived();
     }
 
     public B setAutorenewPollMessage(VKey<PollMessage.Autorenew> autorenewPollMessage) {
       getInstance().autorenewPollMessage = autorenewPollMessage;
+      getInstance().autorenewPollMessageHistoryId = getHistoryId(autorenewPollMessage);
       return thisCastToDerived();
     }
 

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -14,19 +14,25 @@
 
 package google.registry.model.domain;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.annotation.Ignore;
-import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.io.Serializable;
+import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.persistence.Access;
@@ -37,12 +43,12 @@ import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
+import javax.persistence.FetchType;
 import javax.persistence.Id;
 import javax.persistence.IdClass;
 import javax.persistence.Index;
 import javax.persistence.JoinColumn;
+import javax.persistence.JoinColumns;
 import javax.persistence.JoinTable;
 import javax.persistence.OneToMany;
 import javax.persistence.PostLoad;
@@ -66,12 +72,24 @@ import javax.persistence.Table;
 @EntitySubclass
 @Access(AccessType.FIELD)
 @IdClass(DomainHistoryId.class)
-public class DomainHistory extends HistoryEntry {
+public class DomainHistory extends HistoryEntry implements SqlEntity {
 
   // Store DomainContent instead of DomainBase so we don't pick up its @Id
-  DomainContent domainContent;
+  @Nullable DomainContent domainContent;
 
-  @Id String domainRepoId;
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setDomainRepoId(String domainRepoId) {
+    parent = Key.create(DomainBase.class, domainRepoId);
+  }
 
   // We could have reused domainContent.nsHosts here, but Hibernate throws a weird exception after
   // we change to use a composite primary key.
@@ -82,6 +100,24 @@ public class DomainHistory extends HistoryEntry {
   @Column(name = "host_repo_id")
   Set<VKey<HostResource>> nsHosts;
 
+  @OneToMany(
+      cascade = {CascadeType.ALL},
+      fetch = FetchType.EAGER,
+      orphanRemoval = true)
+  @JoinColumns({
+    @JoinColumn(
+        name = "domainHistoryRevisionId",
+        referencedColumnName = "historyRevisionId",
+        insertable = false,
+        updatable = false),
+    @JoinColumn(
+        name = "domainRepoId",
+        referencedColumnName = "domainRepoId",
+        insertable = false,
+        updatable = false)
+  })
+  Set<DomainDsDataHistory> dsDataHistories;
+
   @Override
   @Nullable
   @Access(AccessType.PROPERTY)
@@ -116,18 +152,27 @@ public class DomainHistory extends HistoryEntry {
    *
    * <p>This will be empty for any DomainHistory/HistoryEntry generated before this field was added,
    * mid-2017, as well as any action that does not generate billable events (e.g. updates).
+   *
+   * <p>This method is dedicated for Hibernate, external caller should use {@link
+   * #getDomainTransactionRecords()}.
    */
   @Access(AccessType.PROPERTY)
   @OneToMany(cascade = {CascadeType.ALL})
   @JoinColumn(name = "historyRevisionId", referencedColumnName = "historyRevisionId")
   @JoinColumn(name = "domainRepoId", referencedColumnName = "domainRepoId")
-  @Override
-  public Set<DomainTransactionRecord> getDomainTransactionRecords() {
-    return super.getDomainTransactionRecords();
+  @SuppressWarnings("unused")
+  private Set<DomainTransactionRecord> getInternalDomainTransactionRecords() {
+    return domainTransactionRecords;
+  }
+
+  /** Sets the domain transaction records. This method is dedicated for Hibernate. */
+  @SuppressWarnings("unused")
+  private void setInternalDomainTransactionRecords(
+      Set<DomainTransactionRecord> domainTransactionRecords) {
+    this.domainTransactionRecords = domainTransactionRecords;
   }
 
   @Id
-  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "TempHistorySequenceGenerator")
   @Column(name = "historyRevisionId")
   @Access(AccessType.PROPERTY)
   @Override
@@ -140,27 +185,54 @@ public class DomainHistory extends HistoryEntry {
     return nsHosts;
   }
 
-  /** The state of the {@link DomainContent} object at this point in time. */
-  public DomainContent getDomainContent() {
-    return domainContent;
+  /** Returns the collection of {@link DomainDsDataHistory} instances. */
+  public ImmutableSet<DomainDsDataHistory> getDsDataHistories() {
+    return nullToEmptyImmutableCopy(dsDataHistories);
+  }
+
+  /**
+   * The values of all the fields on the {@link DomainContent} object after the action represented
+   * by this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<DomainContent> getDomainContent() {
+    return Optional.ofNullable(domainContent);
   }
 
   /** The key to the {@link DomainBase} this is based off of. */
-  public VKey<DomainBase> getDomainRepoId() {
-    return VKey.create(DomainBase.class, domainRepoId, Key.create(DomainBase.class, domainRepoId));
+  public VKey<DomainBase> getParentVKey() {
+    return VKey.create(DomainBase.class, getDomainRepoId());
   }
 
+  /** Creates a {@link VKey} instance for this entity. */
   public VKey<DomainHistory> createVKey() {
-    return VKey.createSql(DomainHistory.class, new DomainHistoryId(domainRepoId, getId()));
+    return VKey.create(
+        DomainHistory.class, new DomainHistoryId(getDomainRepoId(), getId()), Key.create(this));
   }
 
   @PostLoad
   void postLoad() {
     if (domainContent != null) {
       domainContent.nsHosts = nullToEmptyImmutableCopy(nsHosts);
+      // Normally Hibernate would see that the domain fields are all null and would fill
+      // domainContent with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+      if (domainContent.getDomainName() == null) {
+        domainContent = null;
+      } else {
+        if (domainContent.getRepoId() == null) {
+          domainContent = domainContent.asBuilder().setRepoId(parent.getName()).build();
+        }
+      }
     }
   }
 
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
   /** Class to represent the composite primary key of {@link DomainHistory} entity. */
   static class DomainHistoryId extends ImmutableObject implements Serializable {
 
@@ -176,19 +248,45 @@ public class DomainHistory extends HistoryEntry {
       this.id = id;
     }
 
-    String getDomainRepoId() {
+    /**
+     * Returns the domain repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getDomainRepoId() {
       return domainRepoId;
     }
 
-    void setDomainRepoId(String domainRepoId) {
-      this.domainRepoId = domainRepoId;
-    }
-
-    long getId() {
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
       return id;
     }
 
-    void setId(long id) {
+    /**
+     * Sets the domain repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setDomainRepoId(String domainRepoId) {
+      this.domainRepoId = domainRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
       this.id = id;
     }
   }
@@ -208,24 +306,29 @@ public class DomainHistory extends HistoryEntry {
 
     public Builder setDomainContent(DomainContent domainContent) {
       getInstance().domainContent = domainContent;
-      if (domainContent != null) {
-        getInstance().nsHosts = nullToEmptyImmutableCopy(domainContent.nsHosts);
-      }
       return this;
     }
 
     public Builder setDomainRepoId(String domainRepoId) {
-      getInstance().domainRepoId = domainRepoId;
       getInstance().parent = Key.create(DomainBase.class, domainRepoId);
       return this;
     }
 
-    // We can remove this once all HistoryEntries are converted to History objects
     @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().domainRepoId = parent.getName();
-      return this;
+    public DomainHistory build() {
+      DomainHistory instance = super.build();
+      // TODO(b/171990736): Assert instance.domainContent is not null after database migration.
+      // Note that we cannot assert that instance.domainContent is not null here because this
+      // builder is also used to convert legacy HistoryEntry objects to DomainHistory, when
+      // domainContent is not available.
+      if (instance.domainContent != null) {
+        instance.nsHosts = nullToEmptyImmutableCopy(instance.domainContent.nsHosts);
+        instance.dsDataHistories =
+            nullToEmptyImmutableCopy(instance.domainContent.getDsData()).stream()
+                .map(dsData -> DomainDsDataHistory.createFrom(instance.id, dsData))
+                .collect(toImmutableSet());
+      }
+      return instance;
     }
   }
 }

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -21,7 +21,9 @@ import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.model.ofy.ObjectifyService;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import javax.annotation.Nullable;
 import javax.persistence.Entity;
 import javax.persistence.Index;
@@ -37,7 +39,7 @@ import org.joda.time.DateTime;
 @Embed
 @Entity
 @Table(indexes = @Index(columnList = "domainRepoId"))
-public class GracePeriod extends GracePeriodBase {
+public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntity {
 
   private static GracePeriod createInternal(
       GracePeriodStatus type,
@@ -52,12 +54,15 @@ public class GracePeriod extends GracePeriodBase {
         (billingEventRecurring != null) == GracePeriodStatus.AUTO_RENEW.equals(type),
         "Recurring billing events must be present on (and only on) autorenew grace periods");
     GracePeriod instance = new GracePeriod();
+    instance.id = ObjectifyService.allocateId();
     instance.type = checkArgumentNotNull(type);
     instance.domainRepoId = checkArgumentNotNull(domainRepoId);
     instance.expirationTime = checkArgumentNotNull(expirationTime);
     instance.clientId = checkArgumentNotNull(clientId);
     instance.billingEventOneTime = billingEventOneTime;
+    instance.billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
     instance.billingEventRecurring = billingEventRecurring;
+    instance.billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
     return instance;
   }
 
@@ -107,14 +112,28 @@ public class GracePeriod extends GracePeriodBase {
   }
 
   /**
-   * Returns a clone of this {@link GracePeriod} with {@link #domainRepoId} set to the given value.
+   * Returns a clone of this {@link GracePeriod} with {@link #domainRepoId} set to the given value
+   * and reconstructed history ids.
    *
    * <p>TODO(b/162739503): Remove this function after fully migrating to Cloud SQL.
    */
-  public GracePeriod cloneWithDomainRepoId(String domainRepoId) {
+  public GracePeriod cloneAfterOfyLoad(String domainRepoId) {
     GracePeriod clone = clone(this);
+    clone.id = ObjectifyService.allocateId();
     clone.domainRepoId = checkArgumentNotNull(domainRepoId);
+    clone.restoreHistoryIds();
     return clone;
   }
 
+  /**
+   * Returns a clone of this {@link GracePeriod} with {@link #billingEventRecurring} set to the
+   * given value.
+   *
+   * <p>TODO(b/162231099): Remove this function after duplicate id issue is solved.
+   */
+  public GracePeriod cloneWithRecurringBillingEvent(VKey<BillingEvent.Recurring> recurring) {
+    GracePeriod clone = clone(this);
+    clone.billingEventRecurring = recurring;
+    return clone;
+  }
 }

core/src/main/java/google/registry/model/domain/GracePeriodBase.java

@@ -14,18 +14,21 @@
 
 package google.registry.model.domain;
 
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.ModelUtils;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.persistence.VKey;
+import java.lang.reflect.Field;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.MappedSuperclass;
 import org.joda.time.DateTime;
 
@@ -36,7 +39,6 @@ public class GracePeriodBase extends ImmutableObject {
 
   /** Unique id required for hibernate representation. */
   @javax.persistence.Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
   @Ignore
   Long id;
 
@@ -67,6 +69,10 @@ public class GracePeriodBase extends ImmutableObject {
   @Column(name = "billing_event_id")
   VKey<OneTime> billingEventOneTime = null;
 
+  @Ignore
+  @Column(name = "billing_event_history_id")
+  Long billingEventOneTimeHistoryId;
+
   /**
    * The recurring billing event corresponding to the action that triggered this grace period, if
    * applicable - i.e. if the action was an autorenew - or null in all other cases.
@@ -75,6 +81,14 @@ public class GracePeriodBase extends ImmutableObject {
   @Column(name = "billing_recurrence_id")
   VKey<BillingEvent.Recurring> billingEventRecurring = null;
 
+  @Ignore
+  @Column(name = "billing_recurrence_history_id")
+  Long billingEventRecurringHistoryId;
+
+  public long getId() {
+    return id;
+  }
+
   public GracePeriodStatus getType() {
     return type;
   }
@@ -101,6 +115,7 @@ public class GracePeriodBase extends ImmutableObject {
    * period is not AUTO_RENEW.
    */
   public VKey<BillingEvent.OneTime> getOneTimeBillingEvent() {
+    restoreOfyKeys();
     return billingEventOneTime;
   }
 
@@ -109,6 +124,63 @@ public class GracePeriodBase extends ImmutableObject {
    * period is AUTO_RENEW.
    */
   public VKey<BillingEvent.Recurring> getRecurringBillingEvent() {
+    restoreOfyKeys();
     return billingEventRecurring;
   }
+
+  /**
+   * Restores history ids for composite VKeys after a load from datastore.
+   *
+   * <p>For use by DomainContent.load() ONLY.
+   */
+  protected void restoreHistoryIds() {
+    billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
+    billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
+  }
+
+  /**
+   * Override {@link ImmutableObject#getSignificantFields()} to exclude "id", which breaks equality
+   * testing in the unit tests.
+   */
+  @Override
+  protected Map<Field, Object> getSignificantFields() {
+    restoreOfyKeys();
+    // Can't use streams or ImmutableMap because we can have null values.
+    Map<Field, Object> result = new LinkedHashMap();
+    for (Map.Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
+      if (!entry.getKey().getName().equals("id")) {
+        result.put(entry.getKey(), entry.getValue());
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Restores Ofy keys in the billing events.
+   *
+   * <p>This must be called by all methods that access the one time or recurring billing event keys.
+   * When the billing event keys are loaded from SQL, they are loaded as asymmetric keys because the
+   * database columns that we load them from do not contain all of the information necessary to
+   * reconsitute the Ofy side of the key. In other cases, we restore the Ofy key during the
+   * hibernate {@link javax.persistence.PostLoad} method from the other fields of the object, but we
+   * have been unable to make this work with hibernate's internal persistence model in this case
+   * because the {@link GracePeriod}'s hash code is evaluated prior to these calls, and would be
+   * invalidated by changing the fields.
+   */
+  private final synchronized void restoreOfyKeys() {
+    if (billingEventOneTime != null && !billingEventOneTime.maybeGetOfyKey().isPresent()) {
+      billingEventOneTime =
+          DomainBase.restoreOfyFrom(
+              Key.create(DomainBase.class, domainRepoId),
+              billingEventOneTime,
+              billingEventOneTimeHistoryId);
+    }
+    if (billingEventRecurring != null && !billingEventRecurring.maybeGetOfyKey().isPresent()) {
+      billingEventRecurring =
+          DomainBase.restoreOfyFrom(
+              Key.create(DomainBase.class, domainRepoId),
+              billingEventRecurring,
+              billingEventRecurringHistoryId);
+    }
+  }
 }

core/src/main/java/google/registry/model/domain/secdns/DelegationSignerData.java

@@ -14,96 +14,235 @@
 
 package google.registry.model.domain.secdns;
 
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.domain.secdns.DelegationSignerData.DomainDsDataId;
+import java.io.Serializable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Index;
+import javax.persistence.Table;
 import javax.xml.bind.DatatypeConverter;
-import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlType;
-import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
-import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 /**
  * Holds the data necessary to construct a single Delegation Signer (DS) record for a domain.
  *
  * @see <a href="http://tools.ietf.org/html/rfc5910">RFC 5910</a>
  * @see <a href="http://tools.ietf.org/html/rfc4034">RFC 4034</a>
+ *     <p>TODO(shicong): Rename this class to DomainDsData.
  */
 @Embed
 @XmlType(name = "dsData")
-@javax.persistence.Entity
-public class DelegationSignerData extends ImmutableObject {
+@Entity
+@IdClass(DomainDsDataId.class)
+@Table(indexes = @Index(columnList = "domainRepoId"))
+public class DelegationSignerData extends DomainDsDataBase {
 
   private DelegationSignerData() {}
 
-  /** The identifier for this particular key in the domain. */
-  @javax.persistence.Id int keyTag;
-
-  /**
-   * The algorithm used by this key.
-   *
-   * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.1">RFC 4034 Appendix A.1</a>
-   */
-  @XmlElement(name = "alg")
-  int algorithm;
-
-  /**
-   * The algorithm used to generate the digest.
-   *
-   * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.2">RFC 4034 Appendix A.2</a>
-   */
-  int digestType;
-
-  /**
-   * The hexBinary digest of the public key.
-   *
-   * @see <a href="http://tools.ietf.org/html/rfc4034#section-5.1.4">RFC 4034 Section 5.1.4</a>
-   */
-  @XmlJavaTypeAdapter(HexBinaryAdapter.class)
-  byte[] digest;
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    return super.getDomainRepoId();
+  }
 
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
   public int getKeyTag() {
-    return keyTag;
+    return super.getKeyTag();
   }
 
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
   public int getAlgorithm() {
-    return algorithm;
+    return super.getAlgorithm();
   }
 
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
   public int getDigestType() {
-    return digestType;
+    return super.getDigestType();
   }
 
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
   public byte[] getDigest() {
-    return digest;
+    return super.getDigest();
   }
 
-  public String getDigestAsString() {
-    return digest == null ? "" : DatatypeConverter.printHexBinary(digest);
+  public DelegationSignerData cloneWithDomainRepoId(String domainRepoId) {
+    DelegationSignerData clone = clone(this);
+    clone.domainRepoId = checkArgumentNotNull(domainRepoId);
+    return clone;
+  }
+
+  public DelegationSignerData cloneWithoutDomainRepoId() {
+    DelegationSignerData clone = clone(this);
+    clone.domainRepoId = null;
+    return clone;
   }
 
   public static DelegationSignerData create(
-      int keyTag, int algorithm, int digestType, byte[] digest) {
+      int keyTag, int algorithm, int digestType, byte[] digest, String domainRepoId) {
     DelegationSignerData instance = new DelegationSignerData();
     instance.keyTag = keyTag;
     instance.algorithm = algorithm;
     instance.digestType = digestType;
     instance.digest = digest;
+    instance.domainRepoId = domainRepoId;
     return instance;
   }
 
+  public static DelegationSignerData create(
+      int keyTag, int algorithm, int digestType, byte[] digest) {
+    return create(keyTag, algorithm, digestType, digest, null);
+  }
+
   public static DelegationSignerData create(
       int keyTag, int algorithm, int digestType, String digestAsHex) {
     return create(keyTag, algorithm, digestType, DatatypeConverter.parseHexBinary(digestAsHex));
   }
 
-  /**
-   * Returns the presentation format of this DS record.
-   *
-   * @see <a href="https://tools.ietf.org/html/rfc4034#section-5.3">RFC 4034 Section 5.3</a>
-   */
-  public String toRrData() {
-    return String.format(
-        "%d %d %d %s",
-        this.keyTag, this.algorithm, this.digestType, DatatypeConverter.printHexBinary(digest));
+  /** Class to represent the composite primary key of {@link DelegationSignerData} entity. */
+  static class DomainDsDataId extends ImmutableObject implements Serializable {
+
+    String domainRepoId;
+
+    int keyTag;
+
+    int algorithm;
+
+    int digestType;
+
+    byte[] digest;
+
+    /** Hibernate requires this default constructor. */
+    private DomainDsDataId() {}
+
+    /** Constructs a {link DomainDsDataId} instance. */
+    DomainDsDataId(String domainRepoId, int keyTag, int algorithm, int digestType, byte[] digest) {
+      this.domainRepoId = domainRepoId;
+      this.keyTag = keyTag;
+      this.algorithm = algorithm;
+      this.digestType = digestType;
+      this.digest = digest;
+    }
+
+    /**
+     * Returns the domain repository ID.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getDomainRepoId() {
+      return domainRepoId;
+    }
+
+    /**
+     * Returns the key tag.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private int getKeyTag() {
+      return keyTag;
+    }
+
+    /**
+     * Returns the algorithm.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private int getAlgorithm() {
+      return algorithm;
+    }
+
+    /**
+     * Returns the digest type.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private int getDigestType() {
+      return digestType;
+    }
+
+    /**
+     * Returns the digest.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private byte[] getDigest() {
+      return digest;
+    }
+
+    /**
+     * Sets the domain repository ID.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setDomainRepoId(String domainRepoId) {
+      this.domainRepoId = domainRepoId;
+    }
+
+    /**
+     * Sets the key tag.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setKeyTag(int keyTag) {
+      this.keyTag = keyTag;
+    }
+
+    /**
+     * Sets the algorithm.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setAlgorithm(int algorithm) {
+      this.algorithm = algorithm;
+    }
+
+    /**
+     * Sets the digest type.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setDigestType(int digestType) {
+      this.digestType = digestType;
+    }
+
+    /**
+     * Sets the digest.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setDigest(byte[] digest) {
+      this.digest = digest;
+    }
+
+    public static DomainDsDataId create(
+        String domainRepoId, int keyTag, int algorithm, int digestType, byte[] digest) {
+      return new DomainDsDataId(
+          domainRepoId, keyTag, algorithm, digestType, checkArgumentNotNull(digest));
+    }
   }
 }

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataBase.java

@@ -0,0 +1,150 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.domain.secdns;
+
+import com.googlecode.objectify.annotation.Embed;
+import com.googlecode.objectify.annotation.Ignore;
+import google.registry.model.ImmutableObject;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.Transient;
+import javax.xml.bind.DatatypeConverter;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+/** Base class for {@link DelegationSignerData} and {@link DomainDsDataHistory}. */
+@Embed
+@MappedSuperclass
+@Access(AccessType.FIELD)
+public abstract class DomainDsDataBase extends ImmutableObject {
+
+  @Ignore @XmlTransient @Transient String domainRepoId;
+
+  /** The identifier for this particular key in the domain. */
+  @Transient int keyTag;
+
+  /**
+   * The algorithm used by this key.
+   *
+   * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.1">RFC 4034 Appendix A.1</a>
+   */
+  @Transient
+  @XmlElement(name = "alg")
+  int algorithm;
+
+  /**
+   * The algorithm used to generate the digest.
+   *
+   * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.2">RFC 4034 Appendix A.2</a>
+   */
+  @Transient int digestType;
+
+  /**
+   * The hexBinary digest of the public key.
+   *
+   * @see <a href="http://tools.ietf.org/html/rfc4034#section-5.1.4">RFC 4034 Section 5.1.4</a>
+   */
+  @Transient
+  @XmlJavaTypeAdapter(HexBinaryAdapter.class)
+  byte[] digest;
+
+  public String getDomainRepoId() {
+    return domainRepoId;
+  }
+
+  public int getKeyTag() {
+    return keyTag;
+  }
+
+  public int getAlgorithm() {
+    return algorithm;
+  }
+
+  public int getDigestType() {
+    return digestType;
+  }
+
+  public byte[] getDigest() {
+    return digest;
+  }
+
+  /**
+   * Sets the domain repository ID.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setDomainRepoId(String domainRepoId) {
+    this.domainRepoId = domainRepoId;
+  }
+
+  /**
+   * Sets the key tag.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setKeyTag(int keyTag) {
+    this.keyTag = keyTag;
+  }
+
+  /**
+   * Sets the algorithm.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setAlgorithm(int algorithm) {
+    this.algorithm = algorithm;
+  }
+
+  /**
+   * Sets the digest type.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setDigestType(int digestType) {
+    this.digestType = digestType;
+  }
+
+  /**
+   * Sets the digest.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setDigest(byte[] digest) {
+    this.digest = digest;
+  }
+
+  public String getDigestAsString() {
+    return digest == null ? "" : DatatypeConverter.printHexBinary(digest);
+  }
+
+  /**
+   * Returns the presentation format of this DS record.
+   *
+   * @see <a href="https://tools.ietf.org/html/rfc4034#section-5.3">RFC 4034 Section 5.3</a>
+   */
+  public String toRrData() {
+    return String.format(
+        "%d %d %d %s",
+        this.keyTag, this.algorithm, this.digestType, DatatypeConverter.printHexBinary(digest));
+  }
+}

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java

@@ -0,0 +1,92 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.domain.secdns;
+
+import com.google.common.collect.ImmutableList;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.ofy.ObjectifyService;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+
+/** Entity class to represent a historic {@link DelegationSignerData}. */
+@Entity
+public class DomainDsDataHistory extends DomainDsDataBase implements SqlEntity {
+
+  @Id Long dsDataHistoryRevisionId;
+
+  /** ID of the {@link DomainHistory} entity that this entity is associated with. */
+  @Column(nullable = false)
+  Long domainHistoryRevisionId;
+
+  private DomainDsDataHistory() {}
+
+  /**
+   * Creates a {@link DomainDsDataHistory} instance from given {@link #domainHistoryRevisionId} and
+   * {@link DelegationSignerData} instance.
+   */
+  public static DomainDsDataHistory createFrom(
+      long domainHistoryRevisionId, DelegationSignerData dsData) {
+    DomainDsDataHistory instance = new DomainDsDataHistory();
+    instance.domainHistoryRevisionId = domainHistoryRevisionId;
+    instance.domainRepoId = dsData.domainRepoId;
+    instance.keyTag = dsData.getKeyTag();
+    instance.algorithm = dsData.getAlgorithm();
+    instance.digestType = dsData.getDigestType();
+    instance.digest = dsData.getDigest();
+    instance.dsDataHistoryRevisionId = ObjectifyService.allocateId();
+    return instance;
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    return super.getDomainRepoId();
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  public int getKeyTag() {
+    return super.getKeyTag();
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  public int getAlgorithm() {
+    return super.getAlgorithm();
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  public int getDigestType() {
+    return super.getDigestType();
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  @Column(nullable = false)
+  public byte[] getDigest() {
+    return super.getDigest();
+  }
+
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(); // not persisted in Datastore
+  }
+}

core/src/main/java/google/registry/model/eppcommon/StatusValue.java

@@ -20,8 +20,10 @@ import static com.google.common.base.Strings.nullToEmpty;
 
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactBase;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
 import google.registry.model.host.HostBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.translators.EnumToAttributeAdapter.EppEnum;
@@ -128,9 +130,15 @@ public enum StatusValue implements EppEnum {
 
   /** Enum to help clearly list which resource types a status value is allowed to be present on. */
   private enum AllowedOn {
-    ALL(ContactResource.class, DomainBase.class, HostBase.class, HostResource.class),
+    ALL(
+        ContactBase.class,
+        ContactResource.class,
+        DomainContent.class,
+        DomainBase.class,
+        HostBase.class,
+        HostResource.class),
     NONE,
-    DOMAINS(DomainBase.class);
+    DOMAINS(DomainContent.class, DomainBase.class);
 
     private final ImmutableSet<Class<? extends EppResource>> classes;
 

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -14,18 +14,25 @@
 
 package google.registry.model.host;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
-import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.host.HostHistory.HostHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
 
 /**
  * A persisted history entry representing an EPP modification to a host.
@@ -45,16 +52,27 @@ import javax.persistence.Id;
     })
 @EntitySubclass
 @Access(AccessType.FIELD)
-public class HostHistory extends HistoryEntry {
+@IdClass(HostHistoryId.class)
+public class HostHistory extends HistoryEntry implements SqlEntity {
 
   // Store HostBase instead of HostResource so we don't pick up its @Id
-  HostBase hostBase;
-
-  @Column(nullable = false)
-  VKey<HostResource> hostRepoId;
+  @Nullable HostBase hostBase;
+
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getHostRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setHostRepoId(String hostRepoId) {
+    parent = Key.create(HostResource.class, hostRepoId);
+  }
 
   @Id
-  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "TempHistorySequenceGenerator")
   @Column(name = "historyRevisionId")
   @Access(AccessType.PROPERTY)
   @Override
@@ -62,14 +80,98 @@ public class HostHistory extends HistoryEntry {
     return super.getId();
   }
 
-  /** The state of the {@link HostBase} object at this point in time. */
-  public HostBase getHostBase() {
-    return hostBase;
+  /**
+   * The values of all the fields on the {@link HostBase} object after the action represented by
+   * this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<HostBase> getHostBase() {
+    return Optional.ofNullable(hostBase);
   }
 
   /** The key to the {@link google.registry.model.host.HostResource} this is based off of. */
-  public VKey<HostResource> getHostRepoId() {
-    return hostRepoId;
+  public VKey<HostResource> getParentVKey() {
+    return VKey.create(HostResource.class, getHostRepoId());
+  }
+
+  /** Creates a {@link VKey} instance for this entity. */
+  public VKey<HostHistory> createVKey() {
+    return VKey.create(
+        HostHistory.class, new HostHistoryId(getHostRepoId(), getId()), Key.create(this));
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Normally Hibernate would see that the host fields are all null and would fill hostBase
+    // with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+    if (hostBase != null && hostBase.getHostName() == null) {
+      hostBase = null;
+    }
+  }
+
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
+  /** Class to represent the composite primary key of {@link HostHistory} entity. */
+  static class HostHistoryId extends ImmutableObject implements Serializable {
+
+    private String hostRepoId;
+
+    private Long id;
+
+    /** Hibernate requires this default constructor. */
+    private HostHistoryId() {}
+
+    HostHistoryId(String hostRepoId, long id) {
+      this.hostRepoId = hostRepoId;
+      this.id = id;
+    }
+
+    /**
+     * Returns the host repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getHostRepoId() {
+      return hostRepoId;
+    }
+
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
+      return id;
+    }
+
+    /**
+     * Sets the host repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setHostRepoId(String hostRepoId) {
+      this.hostRepoId = hostRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
+      this.id = id;
+    }
   }
 
   @Override
@@ -90,18 +192,8 @@ public class HostHistory extends HistoryEntry {
       return this;
     }
 
-    public Builder setHostRepoId(VKey<HostResource> hostRepoId) {
-      getInstance().hostRepoId = hostRepoId;
-      hostRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
-      return this;
-    }
-
-    // We can remove this once all HistoryEntries are converted to History objects
-    @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().hostRepoId =
-          VKey.create(HostResource.class, parent.getName(), (Key<HostResource>) parent);
+    public Builder setHostRepoId(String hostRepoId) {
+      getInstance().parent = Key.create(HostResource.class, hostRepoId);
       return this;
     }
   }

core/src/main/java/google/registry/model/host/HostResource.java

@@ -32,7 +32,7 @@ import javax.persistence.AccessType;
  */
 @ReportedOn
 @Entity
-@javax.persistence.Entity
+@javax.persistence.Entity(name = "Host")
 @ExternalMessagingName("host")
 @WithStringVKey
 @Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property)

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -20,7 +20,6 @@ import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntri
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.TypeUtils.instantiate;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
@@ -244,7 +243,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   private static LoadingCache<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>>
       createForeignKeyIndexesCache(Duration expiry) {
     return CacheBuilder.newBuilder()
-        .expireAfterWrite(expiry.getMillis(), MILLISECONDS)
+        .expireAfterWrite(java.time.Duration.ofMillis(expiry.getMillis()))
         .maximumSize(getEppResourceMaxCachedEntries())
         .build(CACHE_LOADER);
   }

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -23,7 +23,9 @@ import com.google.common.base.Functions;
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
+import com.googlecode.objectify.Result;
 import google.registry.model.contact.ContactHistory;
 import google.registry.model.host.HostHistory;
 import google.registry.model.reporting.HistoryEntry;
@@ -101,42 +103,72 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public void saveNew(Object entity) {
+  public void insert(Object entity) {
+    put(entity);
+  }
+
+  @Override
+  public void insertAll(ImmutableCollection<?> entities) {
+    putAll(entities);
+  }
+
+  @Override
+  public void insertWithoutBackup(Object entity) {
+    putWithoutBackup(entity);
+  }
+
+  @Override
+  public void insertAllWithoutBackup(ImmutableCollection<?> entities) {
+    putAllWithoutBackup(entities);
+  }
+
+  @Override
+  public void put(Object entity) {
     saveEntity(entity);
   }
 
   @Override
-  public void saveAllNew(ImmutableCollection<?> entities) {
-    getOfy().save().entities(entities);
+  public void putAll(ImmutableCollection<?> entities) {
+    syncIfTransactionless(getOfy().save().entities(entities));
   }
 
   @Override
-  public void saveNewOrUpdate(Object entity) {
-    saveEntity(entity);
+  public void putWithoutBackup(Object entity) {
+    syncIfTransactionless(getOfy().saveWithoutBackup().entities(entity));
   }
 
   @Override
-  public void saveNewOrUpdateAll(ImmutableCollection<?> entities) {
-    getOfy().save().entities(entities);
+  public void putAllWithoutBackup(ImmutableCollection<?> entities) {
+    syncIfTransactionless(getOfy().saveWithoutBackup().entities(entities));
   }
 
   @Override
   public void update(Object entity) {
-    saveEntity(entity);
+    put(entity);
   }
 
   @Override
   public void updateAll(ImmutableCollection<?> entities) {
-    getOfy().save().entities(entities);
+    putAll(entities);
   }
 
   @Override
-  public boolean checkExists(Object entity) {
+  public void updateWithoutBackup(Object entity) {
+    putWithoutBackup(entity);
+  }
+
+  @Override
+  public void updateAllWithoutBackup(ImmutableCollection<?> entities) {
+    putAllWithoutBackup(entities);
+  }
+
+  @Override
+  public boolean exists(Object entity) {
     return getOfy().load().key(Key.create(entity)).now() != null;
   }
 
   @Override
-  public <T> boolean checkExists(VKey<T> key) {
+  public <T> boolean exists(VKey<T> key) {
     return loadNullable(key) != null;
   }
 
@@ -158,6 +190,11 @@ public class DatastoreTransactionManager implements TransactionManager {
     return result;
   }
 
+  @Override
+  public <T> T load(T entity) {
+    return ofy().load().entity(entity).now();
+  }
+
   @Override
   public <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys) {
     // Keep track of the Key -> VKey mapping so we can translate them back.
@@ -175,13 +212,17 @@ public class DatastoreTransactionManager implements TransactionManager {
 
   @Override
   public <T> ImmutableList<T> loadAll(Class<T> clazz) {
-    // We can do a ofy().load().type(clazz), but this doesn't work in a transaction.
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    return ImmutableList.copyOf(getOfy().load().type(clazz));
+  }
+
+  @Override
+  public <T> ImmutableList<T> loadAll(Iterable<T> entities) {
+    return ImmutableList.copyOf(getOfy().load().entities(entities).values());
   }
 
   @Override
   public void delete(VKey<?> key) {
-    getOfy().delete().key(key.getOfyKey()).now();
+    syncIfTransactionless(getOfy().delete().key(key.getOfyKey()));
   }
 
   @Override
@@ -192,7 +233,35 @@ public class DatastoreTransactionManager implements TransactionManager {
         StreamSupport.stream(vKeys.spliterator(), false)
             .map(VKey::getOfyKey)
             .collect(toImmutableList());
-    getOfy().delete().keys(list).now();
+    syncIfTransactionless(getOfy().delete().keys(list));
+  }
+
+  @Override
+  public void delete(Object entity) {
+    syncIfTransactionless(getOfy().delete().entity(entity));
+  }
+
+  @Override
+  public void deleteWithoutBackup(VKey<?> key) {
+    syncIfTransactionless(getOfy().deleteWithoutBackup().key(key.getOfyKey()));
+  }
+
+  @Override
+  public void deleteWithoutBackup(Iterable<? extends VKey<?>> keys) {
+    syncIfTransactionless(
+        getOfy()
+            .deleteWithoutBackup()
+            .keys(Streams.stream(keys).map(VKey::getOfyKey).collect(toImmutableList())));
+  }
+
+  @Override
+  public void deleteWithoutBackup(Object entity) {
+    syncIfTransactionless(getOfy().deleteWithoutBackup().entity(entity));
+  }
+
+  @Override
+  public void clearSessionCache() {
+    getOfy().clearSessionCache();
   }
 
   /**
@@ -209,7 +278,7 @@ public class DatastoreTransactionManager implements TransactionManager {
     if (entity instanceof HistoryEntry) {
       entity = ((HistoryEntry) entity).asHistoryEntry();
     }
-    getOfy().save().entity(entity);
+    syncIfTransactionless(getOfy().save().entity(entity));
   }
 
   @SuppressWarnings("unchecked")
@@ -227,4 +296,19 @@ public class DatastoreTransactionManager implements TransactionManager {
   private <T> T loadNullable(VKey<T> key) {
     return toChildHistoryEntryIfPossible(getOfy().load().key(key.getOfyKey()).now());
   }
+
+  /**
+   * Executes the given {@link Result} instance synchronously if not in a transaction.
+   *
+   * <p>The {@link Result} instance contains a task that will be executed by Objectify
+   * asynchronously. If it is in a transaction, we don't need to execute the task immediately
+   * because it is guaranteed to be done by the end of the transaction. However, if it is not in a
+   * transaction, we need to execute it in case the following code expects that happens before
+   * themselves.
+   */
+  private void syncIfTransactionless(Result<?> result) {
+    if (!inTransaction()) {
+      result.now();
+    }
+  }
 }

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -27,14 +27,18 @@ import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainRenewData;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
+import google.registry.model.host.HostResource;
 import google.registry.model.poll.PendingActionNotificationResponse.ContactPendingActionNotificationResponse;
 import google.registry.model.poll.PendingActionNotificationResponse.DomainPendingActionNotificationResponse;
 import google.registry.model.poll.PendingActionNotificationResponse.HostPendingActionNotificationResponse;
@@ -54,10 +58,9 @@ import javax.persistence.Column;
 import javax.persistence.DiscriminatorColumn;
 import javax.persistence.DiscriminatorValue;
 import javax.persistence.Embedded;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
+import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
@@ -98,7 +101,6 @@ public abstract class PollMessage extends ImmutableObject
   /** Entity id. */
   @Id
   @javax.persistence.Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
   @Column(name = "poll_message_id")
   Long id;
 
@@ -124,11 +126,11 @@ public abstract class PollMessage extends ImmutableObject
 
   @Ignore String hostRepoId;
 
-  @Ignore Long domainRevisionId;
+  @Ignore Long domainHistoryRevisionId;
 
-  @Ignore Long contactRevisionId;
+  @Ignore Long contactHistoryRevisionId;
 
-  @Ignore Long hostRevisionId;
+  @Ignore Long hostHistoryRevisionId;
 
   public Key<HistoryEntry> getParentKey() {
     return parent;
@@ -152,6 +154,34 @@ public abstract class PollMessage extends ImmutableObject
 
   public abstract ImmutableList<ResponseData> getResponseData();
 
+  @PostLoad
+  void postLoad() {
+    if (domainRepoId != null) {
+      parent =
+          Key.create(
+              Key.create(DomainBase.class, domainRepoId),
+              HistoryEntry.class,
+              domainHistoryRevisionId);
+    } else if (contactRepoId != null) {
+      parent =
+          Key.create(
+              Key.create(ContactResource.class, contactRepoId),
+              HistoryEntry.class,
+              contactHistoryRevisionId);
+    } else if (hostHistoryRevisionId != null) {
+      parent =
+          Key.create(
+              Key.create(HostResource.class, hostRepoId),
+              HistoryEntry.class,
+              hostHistoryRevisionId);
+    }
+  }
+
+  @OnLoad
+  void onLoad() {
+    setSqlForeignKeys(this);
+  }
+
   @Override
   public abstract VKey<? extends PollMessage> createVKey();
 
@@ -217,10 +247,30 @@ public abstract class PollMessage extends ImmutableObject
       checkArgumentNotNull(instance.clientId, "clientId must be specified");
       checkArgumentNotNull(instance.eventTime, "eventTime must be specified");
       checkArgumentNotNull(instance.parent, "parent must be specified");
+      checkArgumentNotNull(instance.parent.getParent(), "parent.getParent() must be specified");
+      setSqlForeignKeys(instance);
       return super.build();
     }
   }
 
+  private static void setSqlForeignKeys(PollMessage pollMessage) {
+    String grandparentKind = pollMessage.parent.getParent().getKind();
+    String repoId = pollMessage.parent.getParent().getName();
+    long historyRevisionId = pollMessage.parent.getId();
+    if (Key.getKind(DomainBase.class).equals(grandparentKind)) {
+      pollMessage.domainRepoId = repoId;
+      pollMessage.domainHistoryRevisionId = historyRevisionId;
+    } else if (Key.getKind(ContactResource.class).equals(grandparentKind)) {
+      pollMessage.contactRepoId = repoId;
+      pollMessage.contactHistoryRevisionId = historyRevisionId;
+    } else if (Key.getKind(HostResource.class).equals(grandparentKind)) {
+      pollMessage.hostRepoId = repoId;
+      pollMessage.hostHistoryRevisionId = historyRevisionId;
+    } else {
+      throw new IllegalArgumentException("Unknown grandparent kind: " + grandparentKind);
+    }
+  }
+
   /**
    * A one-time poll message.
    *

core/src/main/java/google/registry/model/rde/RdeRevision.java

@@ -15,17 +15,32 @@
 package google.registry.model.rde;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Verify.verify;
-import static com.google.common.base.Verify.verifyNotNull;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.rde.RdeNamingUtils.makePartialName;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.base.VerifyException;
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
+import google.registry.model.BackupGroupRoot;
 import google.registry.model.ImmutableObject;
+import google.registry.model.rde.RdeRevision.RdeRevisionId;
+import google.registry.persistence.VKey;
+import google.registry.persistence.converter.LocalDateConverter;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.persistence.Column;
+import javax.persistence.Convert;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.IdClass;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
+import org.joda.time.LocalDate;
 
 /**
  * Datastore entity for tracking RDE revisions.
@@ -35,32 +50,67 @@ import org.joda.time.DateTime;
  * flag is included in the generated XML.
  */
 @Entity
-public final class RdeRevision extends ImmutableObject {
+@javax.persistence.Entity
+@IdClass(RdeRevisionId.class)
+public final class RdeRevision extends BackupGroupRoot implements DatastoreEntity, SqlEntity {
 
   /** String triplet of tld, date, and mode, e.g. {@code soy_2015-09-01_full}. */
-  @Id
-  String id;
+  @Id @Transient String id;
+
+  @javax.persistence.Id @Ignore String tld;
+
+  @javax.persistence.Id @Ignore LocalDate date;
+
+  @javax.persistence.Id @Ignore RdeMode mode;
 
   /**
    * Number of last revision successfully staged to GCS.
    *
    * <p>This values begins at zero upon object creation and thenceforth incremented transactionally.
    */
+  @Column(nullable = false)
   int revision;
 
+  /** Hibernate requires an empty constructor. */
+  private RdeRevision() {}
+
+  public static RdeRevision create(
+      String id, String tld, LocalDate date, RdeMode mode, int revision) {
+    RdeRevision instance = new RdeRevision();
+    instance.id = id;
+    instance.tld = tld;
+    instance.date = date;
+    instance.mode = mode;
+    instance.revision = revision;
+    return instance;
+  }
+
   public int getRevision() {
     return revision;
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // we don't care about RdeRevision history
+  }
+
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(); // we don't care about RdeRevision history
+  }
+
   /**
    * Returns next revision ID to use when staging a new deposit file for the given triplet.
    *
    * @return {@code 0} for first deposit generation and {@code >0} for resends
    */
   public static int getNextRevision(String tld, DateTime date, RdeMode mode) {
-    RdeRevision object =
-        ofy().load().type(RdeRevision.class).id(makePartialName(tld, date, mode)).now();
-    return object == null ? 0 : object.revision + 1;
+    String id = makePartialName(tld, date, mode);
+    RdeRevisionId sqlKey = RdeRevisionId.create(tld, date.toLocalDate(), mode);
+    Key<RdeRevision> ofyKey = Key.create(RdeRevision.class, id);
+    Optional<RdeRevision> revisionOptional =
+        tm().maybeLoad(VKey.create(RdeRevision.class, sqlKey, ofyKey));
+    return revisionOptional.map(rdeRevision -> rdeRevision.revision + 1).orElse(0);
   }
 
   /**
@@ -76,17 +126,56 @@ public final class RdeRevision extends ImmutableObject {
     checkArgument(revision >= 0, "Negative revision: %s", revision);
     String triplet = makePartialName(tld, date, mode);
     tm().assertInTransaction();
-    RdeRevision object = ofy().load().type(RdeRevision.class).id(triplet).now();
+    RdeRevisionId sqlKey = RdeRevisionId.create(tld, date.toLocalDate(), mode);
+    Key<RdeRevision> ofyKey = Key.create(RdeRevision.class, triplet);
+    Optional<RdeRevision> revisionOptional =
+        tm().maybeLoad(VKey.create(RdeRevision.class, sqlKey, ofyKey));
     if (revision == 0) {
-      verify(object == null, "RdeRevision object already created: %s", object);
+      revisionOptional.ifPresent(
+          rdeRevision -> {
+            throw new IllegalArgumentException(
+                String.format(
+                    "RdeRevision object already created and revision 0 specified: %s",
+                    rdeRevision));
+          });
     } else {
-      verifyNotNull(object, "RDE revision object missing for %s?! revision=%s", triplet, revision);
-      verify(object.revision == revision - 1,
-          "RDE revision object should be at %s but was: %s", revision - 1, object);
+      checkArgument(
+          revisionOptional.isPresent(),
+          "Couldn't find existing RDE revision %s when trying to save new revision %s",
+          triplet,
+          revision);
+      checkArgument(
+          revisionOptional.get().revision == revision - 1,
+          "RDE revision object should be at revision %s but was: %s",
+          revision - 1,
+          revisionOptional.get());
+    }
+    RdeRevision object = RdeRevision.create(triplet, tld, date.toLocalDate(), mode, revision);
+    tm().put(object);
+  }
+
+  /** Class to represent the composite primary key of {@link RdeRevision} entity. */
+  static class RdeRevisionId extends ImmutableObject implements Serializable {
+
+    String tld;
+
+    // Auto-conversion doesn't work for ID classes, we must specify @Column and @Convert
+    @Column(columnDefinition = "date")
+    @Convert(converter = LocalDateConverter.class)
+    LocalDate date;
+
+    @Enumerated(EnumType.STRING)
+    RdeMode mode;
+
+    /** Hibernate requires this default constructor. */
+    private RdeRevisionId() {}
+
+    static RdeRevisionId create(String tld, LocalDate date, RdeMode mode) {
+      RdeRevisionId instance = new RdeRevisionId();
+      instance.tld = tld;
+      instance.date = date;
+      instance.mode = mode;
+      return instance;
     }
-    object = new RdeRevision();
-    object.id = triplet;
-    object.revision = revision;
-    ofy().save().entity(object);
   }
 }

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -32,6 +32,7 @@ import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.registry.Registries.assertTldsExist;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableSortedCopy;
 import static google.registry.util.PasswordUtils.SALT_SUPPLIER;
@@ -704,10 +705,18 @@ public class Registrar extends ImmutableObject
     return new Builder(clone(this));
   }
 
+  /** Creates a {@link VKey} for this instance. */
   public VKey<Registrar> createVKey() {
     return VKey.create(Registrar.class, clientIdentifier, Key.create(this));
   }
 
+  /** Creates a {@link VKey} for the given {@code registrarId}. */
+  public static VKey<Registrar> createVKey(String registrarId) {
+    checkArgumentNotNull(registrarId, "registrarId must be specified");
+    return VKey.create(
+        Registrar.class, registrarId, Key.create(getCrossTldKey(), Registrar.class, registrarId));
+  }
+
   /** A builder for constructing {@link Registrar}, since it is immutable. */
   public static class Builder extends Buildable.Builder<Registrar> {
     public Builder() {}
@@ -991,8 +1000,7 @@ public class Registrar extends ImmutableObject
   /** Loads and returns a registrar entity by its client id directly from Datastore. */
   public static Optional<Registrar> loadByClientId(String clientId) {
     checkArgument(!Strings.isNullOrEmpty(clientId), "clientId must be specified");
-    return Optional.ofNullable(
-        ofy().load().type(Registrar.class).parent(getCrossTldKey()).id(clientId).now());
+    return transactIfJpaTm(() -> tm().maybeLoad(createVKey(clientId)));
   }
 
   /**

core/src/main/java/google/registry/model/registry/Registries.java

@@ -25,6 +25,7 @@ import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.ofyOrJpaTm;
 import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -59,15 +60,21 @@ public final class Registries {
             tm().doTransactionless(
                     () -> {
                       ImmutableSet<String> tlds =
-                          ofy()
-                              .load()
-                              .type(Registry.class)
-                              .ancestor(getCrossTldKey())
-                              .keys()
-                              .list()
-                              .stream()
-                              .map(Key::getName)
-                              .collect(toImmutableSet());
+                          ofyOrJpaTm(
+                              () ->
+                                  ofy()
+                                      .load()
+                                      .type(Registry.class)
+                                      .ancestor(getCrossTldKey())
+                                      .keys()
+                                      .list()
+                                      .stream()
+                                      .map(Key::getName)
+                                      .collect(toImmutableSet()),
+                              () ->
+                                  tm().loadAll(Registry.class).stream()
+                                      .map(Registry::getTldStr)
+                                      .collect(toImmutableSet()));
                       return Registry.getAll(tlds).stream()
                           .map(e -> Maps.immutableEntry(e.getTldStr(), e.getTldType()))
                           .collect(entriesToImmutableMap());

core/src/main/java/google/registry/model/registry/Registry.java

@@ -22,13 +22,11 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Maps.toMap;
 import static google.registry.config.RegistryConfig.getSingletonCacheRefreshDuration;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.joda.money.CurrencyUnit.USD;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -62,6 +60,8 @@ import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Fee;
 import google.registry.model.registry.label.PremiumList;
 import google.registry.model.registry.label.ReservedList;
+import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.util.Idn;
 import java.util.Map;
 import java.util.Optional;
@@ -86,7 +86,7 @@ import org.joda.time.Duration;
 @ReportedOn
 @Entity
 @javax.persistence.Entity(name = "Tld")
-public class Registry extends ImmutableObject implements Buildable {
+public class Registry extends ImmutableObject implements Buildable, DatastoreAndSqlEntity {
 
   @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
@@ -259,35 +259,32 @@ public class Registry extends ImmutableObject implements Buildable {
   /** A cache that loads the {@link Registry} for a given tld. */
   private static final LoadingCache<String, Optional<Registry>> CACHE =
       CacheBuilder.newBuilder()
-          .expireAfterWrite(getSingletonCacheRefreshDuration().getMillis(), MILLISECONDS)
+          .expireAfterWrite(
+              java.time.Duration.ofMillis(getSingletonCacheRefreshDuration().getMillis()))
           .build(
               new CacheLoader<String, Optional<Registry>>() {
                 @Override
                 public Optional<Registry> load(final String tld) {
                   // Enter a transaction-less context briefly; we don't want to enroll every TLD in
                   // a transaction that might be wrapping this call.
-                  return Optional.ofNullable(
-                      tm().doTransactionless(
-                              () ->
-                                  ofy()
-                                      .load()
-                                      .key(Key.create(getCrossTldKey(), Registry.class, tld))
-                                      .now()));
+                  return tm().doTransactionless(() -> tm().maybeLoad(createVKey(tld)));
                 }
 
                 @Override
                 public Map<String, Optional<Registry>> loadAll(Iterable<? extends String> tlds) {
-                  ImmutableMap<String, Key<Registry>> keysMap =
-                      toMap(
-                          ImmutableSet.copyOf(tlds),
-                          tld -> Key.create(getCrossTldKey(), Registry.class, tld));
-                  Map<Key<Registry>, Registry> entities =
-                      tm().doTransactionless(() -> ofy().load().keys(keysMap.values()));
+                  ImmutableMap<String, VKey<Registry>> keysMap =
+                      toMap(ImmutableSet.copyOf(tlds), Registry::createVKey);
+                  Map<VKey<? extends Registry>, Registry> entities =
+                      tm().doTransactionless(() -> tm().load(keysMap.values()));
                   return Maps.transformEntries(
                       keysMap, (k, v) -> Optional.ofNullable(entities.getOrDefault(v, null)));
                 }
               });
 
+  public static VKey<Registry> createVKey(String tld) {
+    return VKey.create(Registry.class, tld, Key.create(getCrossTldKey(), Registry.class, tld));
+  }
+
   /**
    * The name of the pricing engine that this TLD uses.
    *
@@ -385,7 +382,7 @@ public class Registry extends ImmutableObject implements Buildable {
   CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /** The set of reserved lists that are applicable to this registry. */
-  @Column(name = "reserved_list_names", nullable = false)
+  @Column(name = "reserved_list_names")
   Set<Key<ReservedList>> reservedLists;
 
   /** Retrieves an ImmutableSet of all ReservedLists associated with this tld. */

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -25,7 +25,6 @@ import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.allocateId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Splitter;
@@ -197,7 +196,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   @VisibleForTesting
   static LoadingCache<String, PremiumList> createCachePremiumLists(Duration cachePersistDuration) {
     return CacheBuilder.newBuilder()
-        .expireAfterWrite(cachePersistDuration.getMillis(), MILLISECONDS)
+        .expireAfterWrite(java.time.Duration.ofMillis(cachePersistDuration.getMillis()))
         .build(
             new CacheLoader<String, PremiumList>() {
               @Override
@@ -221,7 +220,8 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   static final LoadingCache<Key<PremiumListRevision>, PremiumListRevision>
       cachePremiumListRevisions =
           CacheBuilder.newBuilder()
-              .expireAfterWrite(getSingletonCachePersistDuration().getMillis(), MILLISECONDS)
+              .expireAfterWrite(
+                  java.time.Duration.ofMillis(getSingletonCachePersistDuration().getMillis()))
               .build(
                   new CacheLoader<Key<PremiumListRevision>, PremiumListRevision>() {
                     @Override
@@ -260,14 +260,14 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   static LoadingCache<Key<PremiumListEntry>, Optional<PremiumListEntry>>
       createCachePremiumListEntries(Duration cachePersistDuration) {
     return CacheBuilder.newBuilder()
-        .expireAfterWrite(cachePersistDuration.getMillis(), MILLISECONDS)
+        .expireAfterWrite(java.time.Duration.ofMillis(cachePersistDuration.getMillis()))
         .maximumSize(getStaticPremiumListMaxCachedEntries())
         .build(
             new CacheLoader<Key<PremiumListEntry>, Optional<PremiumListEntry>>() {
               @Override
               public Optional<PremiumListEntry> load(final Key<PremiumListEntry> entryKey) {
-                return tm()
-                    .doTransactionless(() -> Optional.ofNullable(ofy().load().key(entryKey).now()));
+                return tm().doTransactionless(
+                        () -> Optional.ofNullable(ofy().load().key(entryKey).now()));
               }
             });
   }

core/src/main/java/google/registry/model/registry/label/ReservedList.java

@@ -20,7 +20,6 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
 import static google.registry.util.CollectionUtils.nullToEmpty;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.common.base.Splitter;
@@ -241,7 +240,8 @@ public final class ReservedList
 
   private static LoadingCache<String, ReservedList> cache =
       CacheBuilder.newBuilder()
-          .expireAfterWrite(getDomainLabelListCacheDuration().getMillis(), MILLISECONDS)
+          .expireAfterWrite(
+              java.time.Duration.ofMillis(getDomainLabelListCacheDuration().getMillis()))
           .build(
               new CacheLoader<String, ReservedList>() {
                 @Override

core/src/main/java/google/registry/model/registry/label/ReservedListDualWriteDao.java

@@ -43,7 +43,7 @@ public class ReservedListDualWriteDao {
 
   /** Persist a new reserved list to Cloud SQL. */
   public static void save(ReservedList reservedList) {
-    ofyTm().transact(() -> ofyTm().saveNewOrUpdate(reservedList));
+    ofyTm().transact(() -> ofyTm().put(reservedList));
     try {
       logger.atInfo().log("Saving reserved list %s to Cloud SQL", reservedList.getName());
       ReservedListSqlDao.save(reservedList);

core/src/main/java/google/registry/model/registry/label/ReservedListSqlDao.java

@@ -31,7 +31,7 @@ public class ReservedListSqlDao {
   /** Persist a new reserved list to Cloud SQL. */
   public static void save(ReservedList reservedList) {
     checkArgumentNotNull(reservedList, "Must specify reservedList");
-    jpaTm().transact(() -> jpaTm().saveNew(reservedList));
+    jpaTm().transact(() -> jpaTm().insert(reservedList));
   }
 
   /**

core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java

@@ -22,6 +22,7 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.EnumType;
@@ -43,7 +44,8 @@ import org.joda.time.DateTime;
  */
 @Embed
 @Entity
-public class DomainTransactionRecord extends ImmutableObject implements Buildable {
+public class DomainTransactionRecord extends ImmutableObject
+    implements Buildable, DatastoreAndSqlEntity {
 
   @Id
   @Ignore

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -62,7 +62,7 @@ import org.joda.time.DateTime;
 @MappedSuperclass
 @WithStringVKey // TODO(b/162229294): This should be resolved during the course of that bug
 @Access(AccessType.FIELD)
-public class HistoryEntry extends ImmutableObject implements Buildable, DatastoreEntity, SqlEntity {
+public class HistoryEntry extends ImmutableObject implements Buildable, DatastoreEntity {
 
   /** Represents the type of history entry. */
   public enum Type {
@@ -178,7 +178,7 @@ public class HistoryEntry extends ImmutableObject implements Buildable, Datastor
   boolean bySuperuser;
 
   /** Reason for the change. */
-  @Column(nullable = false, name = "historyReason")
+  @Column(name = "historyReason")
   String reason;
 
   /** Whether this change was requested by a registrar. */
@@ -193,7 +193,7 @@ public class HistoryEntry extends ImmutableObject implements Buildable, Datastor
    * transaction counts (such as contact or host mutations).
    */
   @Transient // domain-specific
-  Set<DomainTransactionRecord> domainTransactionRecords;
+  protected Set<DomainTransactionRecord> domainTransactionRecords;
 
   public long getId() {
     // For some reason, Hibernate throws NPE during some initialization phase if we don't deal with
@@ -273,7 +273,8 @@ public class HistoryEntry extends ImmutableObject implements Buildable, Datastor
   /** This method exists solely to satisfy Hibernate. Use the {@link Builder} instead. */
   @SuppressWarnings("UnusedMethod")
   private void setDomainTransactionRecords(Set<DomainTransactionRecord> domainTransactionRecords) {
-    this.domainTransactionRecords = ImmutableSet.copyOf(domainTransactionRecords);
+    this.domainTransactionRecords =
+        domainTransactionRecords == null ? null : ImmutableSet.copyOf(domainTransactionRecords);
   }
 
   public static VKey<HistoryEntry> createVKey(Key<HistoryEntry> key) {
@@ -308,19 +309,10 @@ public class HistoryEntry extends ImmutableObject implements Buildable, Datastor
           new DomainHistory.Builder().copyFrom(this).setDomainRepoId(parent.getName()).build();
     } else if (parentKind.equals(getKind(HostResource.class))) {
       resultEntity =
-          new HostHistory.Builder()
-              .copyFrom(this)
-              .setHostRepoId(
-                  VKey.create(HostResource.class, parent.getName(), (Key<HostResource>) parent))
-              .build();
+          new HostHistory.Builder().copyFrom(this).setHostRepoId(parent.getName()).build();
     } else if (parentKind.equals(getKind(ContactResource.class))) {
       resultEntity =
-          new ContactHistory.Builder()
-              .copyFrom(this)
-              .setContactRepoId(
-                  VKey.create(
-                      ContactResource.class, parent.getName(), (Key<ContactResource>) parent))
-              .build();
+          new ContactHistory.Builder().copyFrom(this).setContactRepoId(parent.getName()).build();
     } else {
       throw new IllegalStateException(
           String.format("Unknown kind of HistoryEntry parent %s", parentKind));
@@ -331,13 +323,7 @@ public class HistoryEntry extends ImmutableObject implements Buildable, Datastor
   // In SQL, save the child type
   @Override
   public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(toChildHistoryEntity());
-  }
-
-  // In Datastore, save as a HistoryEntry object regardless of this object's type
-  @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(asHistoryEntry());
+    return ImmutableList.of((SqlEntity) toChildHistoryEntity());
   }
 
   /** A builder for {@link HistoryEntry} since it is immutable */
@@ -364,7 +350,10 @@ public class HistoryEntry extends ImmutableObject implements Buildable, Datastor
       setBySuperuser(historyEntry.bySuperuser);
       setReason(historyEntry.reason);
       setRequestedByRegistrar(historyEntry.requestedByRegistrar);
-      setDomainTransactionRecords(nullToEmptyImmutableCopy(historyEntry.domainTransactionRecords));
+      setDomainTransactionRecords(
+          historyEntry.domainTransactionRecords == null
+              ? null
+              : ImmutableSet.copyOf(historyEntry.domainTransactionRecords));
       return thisCastToDerived();
     }
 

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java

@@ -75,7 +75,7 @@ public class Spec11ThreatMatch extends ImmutableObject implements Buildable, Sql
   String registrarId;
 
   /** Date on which the check was run, on which the domain was flagged as abusive. */
-  @Column(nullable = false)
+  @Column(nullable = false, columnDefinition = "date")
   LocalDate checkDate;
 
   /** The domain's top-level domain. */

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatchDao.java

@@ -16,23 +16,25 @@ package google.registry.model.reporting;
 
 import com.google.common.collect.ImmutableList;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.util.DateTimeUtils;
+import javax.persistence.TemporalType;
 import org.joda.time.LocalDate;
 
 /**
  * Data access object for {@link google.registry.model.reporting.Spec11ThreatMatch}.
  *
  * <p>A JpaTransactionManager is passed into each static method because they are called from a BEAM
- * pipeline and we don't know where it's coming from.</p>
+ * pipeline and we don't know where it's coming from.
  */
 public class Spec11ThreatMatchDao {
-  
+
   /** Delete all entries with the specified date from the database. */
   public static void deleteEntriesByDate(JpaTransactionManager jpaTm, LocalDate date) {
     jpaTm.assertInTransaction();
     jpaTm
         .getEntityManager()
         .createQuery("DELETE FROM Spec11ThreatMatch WHERE check_date = :date")
-        .setParameter("date", date.toString())
+        .setParameter("date", DateTimeUtils.toSqlDate(date), TemporalType.DATE)
         .executeUpdate();
   }
 

core/src/main/java/google/registry/model/server/KmsSecret.java

@@ -16,6 +16,7 @@ package google.registry.model.server;
 
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -23,11 +24,13 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /** Pointer to the latest {@link KmsSecretRevision}. */
 @Entity
 @ReportedOn
-public class KmsSecret extends ImmutableObject {
+public class KmsSecret extends ImmutableObject implements DatastoreEntity {
 
   /** The unique name of this {@link KmsSecret}. */
   @Id String name;
@@ -45,6 +48,11 @@ public class KmsSecret extends ImmutableObject {
     return latestRevision;
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not persisted in SQL
+  }
+
   public static KmsSecret create(String name, KmsSecretRevision latestRevision) {
     KmsSecret instance = new KmsSecret();
     instance.name = name;

core/src/main/java/google/registry/model/server/KmsSecretRevision.java

@@ -17,14 +17,24 @@ package google.registry.model.server;
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import javax.persistence.Column;
+import javax.persistence.Index;
+import javax.persistence.PostLoad;
+import javax.persistence.Table;
+import javax.persistence.Transient;
 
 /**
  * An encrypted value.
@@ -35,13 +45,22 @@ import google.registry.model.annotations.ReportedOn;
  *
  * <p>The value can be encrypted and decrypted using Cloud KMS.
  *
+ * <p>Note that the primary key of this entity is {@link #revisionKey}, which is auto-generated by
+ * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
+ * succeeds, we will end up with having two exact same revisions that differ only by revisionKey.
+ * This is fine though, because we only use the revision with the highest revisionKey.
+ *
+ * <p>TODO: remove Datastore-specific fields post-Registry-3.0-migration and rename to KmsSecret.
+ *
  * @see <a href="https://cloud.google.com/kms/docs/">Google Cloud Key Management Service
  *     Documentation</a>
  * @see google.registry.keyring.kms.KmsKeyring
  */
 @Entity
 @ReportedOn
-public class KmsSecretRevision extends ImmutableObject {
+@javax.persistence.Entity(name = "KmsSecret")
+@Table(indexes = {@Index(columnList = "secretName")})
+public class KmsSecretRevision extends ImmutableObject implements DatastoreEntity, SqlEntity {
 
   /**
    * The maximum allowable secret size. Although Datastore allows entities up to 1 MB in size,
@@ -49,18 +68,31 @@ public class KmsSecretRevision extends ImmutableObject {
    */
   private static final int MAX_SECRET_SIZE_BYTES = 64 * 1024 * 1024;
 
-  /** The revision of this secret. */
-  @Id long revisionKey;
+  /**
+   * The revision of this secret.
+   *
+   * <p>TODO: change name of the variable to revisionId once we're off Datastore
+   */
+  @Id
+  @javax.persistence.Id
+  @Column(name = "revisionId")
+  long revisionKey;
 
   /** The parent {@link KmsSecret} which contains metadata about this {@link KmsSecretRevision}. */
-  @Parent Key<KmsSecret> parent;
+  @Parent @Transient Key<KmsSecret> parent;
+  @Column(nullable = false)
+  @Ignore
+  String secretName;
 
   /**
    * The name of the {@code cryptoKeyVersion} associated with this {@link KmsSecretRevision}.
    *
+   * <p>TODO: change name of the variable to cryptoKeyVersionName once we're off Datastore
+   *
    * @see <a
    *     href="https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys.cryptoKeyVersions">projects.locations.keyRings.cryptoKeys.cryptoKeyVersions</a>
    */
+  @Column(nullable = false, name = "cryptoKeyVersionName")
   String kmsCryptoKeyVersionName;
 
   /**
@@ -70,9 +102,11 @@ public class KmsSecretRevision extends ImmutableObject {
    * @see <a
    *     href="https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys/encrypt">projects.locations.keyRings.cryptoKeys.encrypt</a>
    */
+  @Column(nullable = false)
   String encryptedValue;
 
   /** An automatically managed creation timestamp. */
+  @Column(nullable = false)
   CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   public String getKmsCryptoKeyVersionName() {
@@ -83,6 +117,28 @@ public class KmsSecretRevision extends ImmutableObject {
     return encryptedValue;
   }
 
+  // When loading from SQL, fill out the Datastore-specific field
+  @PostLoad
+  void postLoad() {
+    parent = Key.create(getCrossTldKey(), KmsSecret.class, secretName);
+  }
+
+  // When loading from Datastore, fill out the SQL-specific field
+  @OnLoad
+  void onLoad() {
+    secretName = parent.getName();
+  }
+
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // This is dually-written, as we do not care about history
+  }
+
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(); // This is dually-written, as we do not care about history
+  }
+
   /** A builder for constructing {@link KmsSecretRevision} entities, since they are immutable. */
   public static class Builder extends Buildable.Builder<KmsSecretRevision> {
 
@@ -108,6 +164,7 @@ public class KmsSecretRevision extends ImmutableObject {
      */
     public Builder setParent(String secretName) {
       getInstance().parent = Key.create(getCrossTldKey(), KmsSecret.class, secretName);
+      getInstance().secretName = secretName;
       return this;
     }
   }

core/src/main/java/google/registry/model/server/KmsSecretRevisionSqlDao.java

@@ -0,0 +1,54 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.server;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+
+import java.util.Optional;
+
+/**
+ * A {@link KmsSecretRevision} DAO for Cloud SQL.
+ *
+ * <p>TODO: Rename this class to KmsSecretDao after migrating to Cloud SQL.
+ */
+public class KmsSecretRevisionSqlDao {
+
+  private KmsSecretRevisionSqlDao() {}
+
+  /** Saves the given KMS secret revision. */
+  public static void save(KmsSecretRevision kmsSecretRevision) {
+    checkArgumentNotNull(kmsSecretRevision, "kmsSecretRevision cannot be null");
+    jpaTm().assertInTransaction();
+    jpaTm().put(kmsSecretRevision);
+  }
+
+  /** Returns the latest revision for the secret name given, or absent if nonexistent. */
+  public static Optional<KmsSecretRevision> getLatestRevision(String secretName) {
+    checkArgument(!isNullOrEmpty(secretName), "secretName cannot be null or empty");
+    jpaTm().assertInTransaction();
+    return jpaTm()
+        .getEntityManager()
+        .createQuery(
+            "FROM KmsSecret ks WHERE ks.revisionKey IN (SELECT MAX(revisionKey) FROM "
+                + "KmsSecret subKs WHERE subKs.secretName = :secretName)",
+            KmsSecretRevision.class)
+        .setParameter("secretName", secretName)
+        .getResultStream()
+        .findFirst();
+  }
+}

core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java

@@ -28,8 +28,12 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EmbedMap;
 import com.googlecode.objectify.annotation.Entity;
@@ -41,8 +45,19 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.EntityGroupRoot;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.util.CollectionUtils;
 import java.util.Map;
+import java.util.Optional;
+import javax.persistence.CollectionTable;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.JoinColumn;
+import javax.persistence.MapKeyColumn;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -56,35 +71,50 @@ import org.joda.time.DateTime;
  * order to avoid exceeding the one megabyte max entity size limit, we'll also be sharding that
  * entity into multiple entities, each entity containing {@value #SHARD_SIZE} rows.
  *
+ * <p>TODO: We can remove the sharding once we have converted entirely to Cloud SQL storage during
+ * the Registry 3.0 migration. Then, the entire table will be stored conceptually as one entity (in
+ * fact in SignedMarkRevocationList and SignedMarkRevocationEntry tables).
+ *
  * @see google.registry.tmch.SmdrlCsvParser
- * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.2">
- *     TMCH functional specifications - SMD Revocation List</a>
+ * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.2">TMCH
+ *     functional specifications - SMD Revocation List</a>
  */
 @Entity
+@javax.persistence.Entity
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-public class SignedMarkRevocationList extends ImmutableObject {
+public class SignedMarkRevocationList extends ImmutableObject
+    implements DatastoreEntity, SqlEntity {
 
-  @VisibleForTesting
-  static final int SHARD_SIZE = 10000;
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  @VisibleForTesting static final int SHARD_SIZE = 10000;
 
   /** Common ancestor for queries. */
-  @Parent
-  Key<EntityGroupRoot> parent = getCrossTldKey();
+  @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
   /** ID for the sharded entity. */
-  @Id
-  long id;
+  @Id @Transient long id;
+
+  @Ignore
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  Long revisionId;
 
   /** Time when this list was last updated, as specified in the first line of the CSV file. */
   DateTime creationTime;
 
   /** A map from SMD IDs to revocation time. */
   @EmbedMap
+  @ElementCollection
+  @CollectionTable(
+      name = "SignedMarkRevocationEntry",
+      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
+  @MapKeyColumn(name = "smdId")
+  @Column(name = "revocationTime", nullable = false)
   Map</*@MatchesPattern("[0-9]+-[0-9]+")*/ String, DateTime> revokes;
 
   /** Indicates that this is a shard rather than a "full" list. */
-  @Ignore
-  boolean isShard;
+  @Ignore @Transient boolean isShard;
 
   /**
    * A cached supplier that fetches the SMDRL shards from Datastore and recombines them into a
@@ -92,32 +122,16 @@ public class SignedMarkRevocationList extends ImmutableObject {
    */
   private static final Supplier<SignedMarkRevocationList> CACHE =
       memoizeWithShortExpiration(
-          () ->
-              tm()
-                  .transactNewReadOnly(
-                      () -> {
-                        Iterable<SignedMarkRevocationList> shards =
-                            ofy()
-                                .load()
-                                .type(SignedMarkRevocationList.class)
-                                .ancestor(getCrossTldKey());
-                        DateTime creationTime =
-                            isEmpty(shards)
-                                ? START_OF_TIME
-                                : checkNotNull(
-                                    Iterables.get(shards, 0).creationTime, "creationTime");
-                        ImmutableMap.Builder<String, DateTime> revokes =
-                            new ImmutableMap.Builder<>();
-                        for (SignedMarkRevocationList shard : shards) {
-                          revokes.putAll(shard.revokes);
-                          checkState(
-                              creationTime.equals(shard.creationTime),
-                              "Inconsistent creation times: %s vs. %s",
-                              creationTime,
-                              shard.creationTime);
-                        }
-                        return create(creationTime, revokes.build());
-                      }));
+          () -> {
+            SignedMarkRevocationList datastoreList = loadFromDatastore();
+            // Also load the list from Cloud SQL, compare the two lists, and log if different.
+            try {
+              loadAndCompareCloudSqlList(datastoreList);
+            } catch (Throwable t) {
+              logger.atSevere().withCause(t).log("Error comparing signed mark revocation lists.");
+            }
+            return datastoreList;
+          });
 
   /** Return a single logical instance that combines all Datastore shards. */
   public static SignedMarkRevocationList get() {
@@ -149,10 +163,39 @@ public class SignedMarkRevocationList extends ImmutableObject {
     return revokes.size();
   }
 
-  /** Save this list to Datastore in sharded form. Returns {@code this}. */
+  /** Save this list to Datastore in sharded form and to Cloud SQL. Returns {@code this}. */
   public SignedMarkRevocationList save() {
-    tm()
-        .transact(
+    saveToDatastore();
+    SignedMarkRevocationListDao.trySave(this);
+    return this;
+  }
+
+  /** Loads the shards from Datastore and combines them into one list. */
+  private static SignedMarkRevocationList loadFromDatastore() {
+    return tm().transactNewReadOnly(
+            () -> {
+              Iterable<SignedMarkRevocationList> shards =
+                  ofy().load().type(SignedMarkRevocationList.class).ancestor(getCrossTldKey());
+              DateTime creationTime =
+                  isEmpty(shards)
+                      ? START_OF_TIME
+                      : checkNotNull(Iterables.get(shards, 0).creationTime, "creationTime");
+              ImmutableMap.Builder<String, DateTime> revokes = new ImmutableMap.Builder<>();
+              for (SignedMarkRevocationList shard : shards) {
+                revokes.putAll(shard.revokes);
+                checkState(
+                    creationTime.equals(shard.creationTime),
+                    "Inconsistent creation times: %s vs. %s",
+                    creationTime,
+                    shard.creationTime);
+              }
+              return create(creationTime, revokes.build());
+            });
+  }
+
+  /** Save this list to Datastore in sharded form. */
+  private SignedMarkRevocationList saveToDatastore() {
+    tm().transact(
             () -> {
               ofy()
                   .deleteWithoutBackup()
@@ -165,8 +208,7 @@ public class SignedMarkRevocationList extends ImmutableObject {
               ofy()
                   .saveWithoutBackup()
                   .entities(
-                      CollectionUtils.partitionMap(revokes, SHARD_SIZE)
-                          .stream()
+                      CollectionUtils.partitionMap(revokes, SHARD_SIZE).stream()
                           .map(
                               shardRevokes -> {
                                 SignedMarkRevocationList shard = create(creationTime, shardRevokes);
@@ -180,6 +222,38 @@ public class SignedMarkRevocationList extends ImmutableObject {
     return this;
   }
 
+  private static void loadAndCompareCloudSqlList(SignedMarkRevocationList datastoreList) {
+    // Lifted with some modifications from ClaimsListShard
+    Optional<SignedMarkRevocationList> maybeCloudSqlList =
+        SignedMarkRevocationListDao.getLatestRevision();
+    if (maybeCloudSqlList.isPresent()) {
+      SignedMarkRevocationList cloudSqlList = maybeCloudSqlList.get();
+      MapDifference<String, DateTime> diff =
+          Maps.difference(datastoreList.revokes, cloudSqlList.revokes);
+      if (!diff.areEqual()) {
+        if (diff.entriesDiffering().size() > 10) {
+          logger.atWarning().log(
+              String.format(
+                  "Unequal SM revocation lists detected, Cloud SQL list with revision id %d has %d"
+                      + " different records than the current Datastore list.",
+                  cloudSqlList.revisionId, diff.entriesDiffering().size()));
+        } else {
+          StringBuilder diffMessage = new StringBuilder("Unequal SM revocation lists detected:\n");
+          diff.entriesDiffering()
+              .forEach(
+                  (label, valueDiff) ->
+                      diffMessage.append(
+                          String.format(
+                              "SMD %s has key %s in Datastore and key %s in Cloud SQL.\n",
+                              label, valueDiff.leftValue(), valueDiff.rightValue())));
+          logger.atWarning().log(diffMessage.toString());
+        }
+      }
+    } else {
+      logger.atWarning().log("Signed mark revocation list in Cloud SQL is empty.");
+    }
+  }
+
   /** As a safety mechanism, fail if someone tries to save this class directly. */
   @OnSave
   void disallowUnshardedSaves() {
@@ -188,6 +262,16 @@ public class SignedMarkRevocationList extends ImmutableObject {
     }
   }
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // Dually-written every day
+  }
+
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(); // Dually-written every day
+  }
+
   /** Exception when trying to directly save a {@link SignedMarkRevocationList} without sharding. */
   public static class UnshardedSaveException extends RuntimeException {}
 }

core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java

@@ -0,0 +1,76 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.smd;
+
+import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Supplier;
+import com.google.common.flogger.FluentLogger;
+import java.util.Optional;
+import javax.persistence.EntityManager;
+
+public class SignedMarkRevocationListDao {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static final Supplier<Optional<SignedMarkRevocationList>> CACHE =
+      memoizeWithShortExpiration(SignedMarkRevocationListDao::getLatestRevision);
+
+  /** Returns the most recent revision of the {@link SignedMarkRevocationList}, from cache. */
+  public static Optional<SignedMarkRevocationList> getLatestRevisionCached() {
+    return CACHE.get();
+  }
+
+  public static Optional<SignedMarkRevocationList> getLatestRevision() {
+    return jpaTm()
+        .transact(
+            () -> {
+              EntityManager em = jpaTm().getEntityManager();
+              Long revisionId =
+                  em.createQuery("SELECT MAX(revisionId) FROM SignedMarkRevocationList", Long.class)
+                      .getSingleResult();
+              return em.createQuery(
+                      "FROM SignedMarkRevocationList smrl LEFT JOIN FETCH smrl.revokes "
+                          + "WHERE smrl.revisionId = :revisionId",
+                      SignedMarkRevocationList.class)
+                  .setParameter("revisionId", revisionId)
+                  .getResultStream()
+                  .findFirst();
+            });
+  }
+
+  /**
+   * Try to save the given {@link SignedMarkRevocationList} into Cloud SQL. If the save fails, the
+   * error will be logged but no exception will be thrown.
+   *
+   * <p>This method is used during the dual-write phase of database migration as Datastore is still
+   * the authoritative database.
+   */
+  static void trySave(SignedMarkRevocationList signedMarkRevocationList) {
+    try {
+      SignedMarkRevocationListDao.save(signedMarkRevocationList);
+      logger.atInfo().log(
+          "Inserted %,d signed mark revocations into Cloud SQL",
+          signedMarkRevocationList.revokes.size());
+    } catch (Throwable e) {
+      logger.atSevere().withCause(e).log("Error inserting signed mark revocations into Cloud SQL");
+    }
+  }
+
+  private static void save(SignedMarkRevocationList signedMarkRevocationList) {
+    jpaTm().transact(() -> jpaTm().getEntityManager().persist(signedMarkRevocationList));
+  }
+}

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -46,6 +46,7 @@ import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.annotations.VirtualEntity;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import google.registry.util.CollectionUtils;
@@ -56,7 +57,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.Callable;
 import javax.annotation.Nullable;
 import javax.persistence.CollectionTable;
 import javax.persistence.Column;
@@ -97,7 +97,7 @@ import org.joda.time.DateTime;
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 @javax.persistence.Entity(name = "ClaimsList")
 @Table
-public class ClaimsListShard extends ImmutableObject implements DatastoreEntity {
+public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlEntity {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -145,8 +145,7 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
 
   private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2);
 
-  private static final Callable<ClaimsListShard> LOADER_CALLABLE =
-      () -> {
+  private static ClaimsListShard loadClaimsListShard() {
         // Find the most recent revision.
         Key<ClaimsListRevision> revisionKey = getCurrentRevision();
 
@@ -245,7 +244,9 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
    */
   private static final Supplier<ClaimsListShard> CACHE =
       memoizeWithShortExpiration(
-          () -> LOADER_RETRIER.callWithRetry(LOADER_CALLABLE, IllegalStateException.class));
+          () ->
+              LOADER_RETRIER.callWithRetry(
+                  ClaimsListShard::loadClaimsListShard, IllegalStateException.class));
 
   /** Returns the revision id of this claims list, or throws exception if it is null. */
   public Long getRevisionId() {
@@ -354,11 +355,6 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
     }
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // ClaimsLists are dually written
-  }
-
   /** Virtual parent entity for claims list shards of a specific revision. */
   @Entity
   @VirtualEntity

core/src/main/java/google/registry/model/transfer/DomainTransferData.java

@@ -14,12 +14,16 @@
 
 package google.registry.model.transfer;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.AlsoLoad;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Unindex;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.Period.Unit;
 import google.registry.model.poll.PollMessage;
@@ -86,6 +90,10 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
   @Column(name = "transfer_billing_event_id")
   VKey<BillingEvent.OneTime> serverApproveBillingEvent;
 
+  @Ignore
+  @Column(name = "transfer_billing_event_history_id")
+  Long serverApproveBillingEventHistoryId;
+
   /**
    * The autorenew billing event that should be associated with this resource after the transfer.
    *
@@ -96,6 +104,10 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
   @Column(name = "transfer_billing_recurrence_id")
   VKey<BillingEvent.Recurring> serverApproveAutorenewEvent;
 
+  @Ignore
+  @Column(name = "transfer_billing_recurrence_history_id")
+  Long serverApproveAutorenewEventHistoryId;
+
   /**
    * The autorenew poll message that should be associated with this resource after the transfer.
    *
@@ -106,11 +118,50 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
   @Column(name = "transfer_autorenew_poll_message_id")
   VKey<PollMessage.Autorenew> serverApproveAutorenewPollMessage;
 
+  @Ignore
+  @Column(name = "transfer_autorenew_poll_message_history_id")
+  Long serverApproveAutorenewPollMessageHistoryId;
+
   @Override
   public Builder copyConstantFieldsToBuilder() {
     return super.copyConstantFieldsToBuilder().setTransferPeriod(this.transferPeriod);
   }
 
+  /**
+   * Restores the set of ofy keys after loading from SQL using the specified {@code rootKey}.
+   *
+   * <p>This is for use by DomainBase/DomainHistory PostLoad methods ONLY.
+   */
+  public void restoreOfyKeys(Key<DomainBase> rootKey) {
+    serverApproveBillingEvent =
+        DomainBase.restoreOfyFrom(
+            rootKey, serverApproveBillingEvent, serverApproveBillingEventHistoryId);
+    serverApproveAutorenewEvent =
+        DomainBase.restoreOfyFrom(
+            rootKey, serverApproveAutorenewEvent, serverApproveAutorenewEventHistoryId);
+    serverApproveAutorenewPollMessage =
+        DomainBase.restoreOfyFrom(
+            rootKey, serverApproveAutorenewPollMessage, serverApproveAutorenewPollMessageHistoryId);
+  }
+
+  @SuppressWarnings("unused") // For Hibernate.
+  private void loadServerApproveBillingEventHistoryId(
+      @AlsoLoad("serverApproveBillingEvent") VKey<BillingEvent.OneTime> val) {
+    serverApproveBillingEventHistoryId = DomainBase.getHistoryId(val);
+  }
+
+  @SuppressWarnings("unused") // For Hibernate.
+  private void loadServerApproveAutorenewEventHistoryId(
+      @AlsoLoad("serverApproveAutorenewEvent") VKey<BillingEvent.Recurring> val) {
+    serverApproveAutorenewEventHistoryId = DomainBase.getHistoryId(val);
+  }
+
+  @SuppressWarnings("unused") // For Hibernate.
+  private void loadServerApproveAutorenewPollMessageHistoryId(
+      @AlsoLoad("serverApproveAutorenewPollMessage") VKey<PollMessage.Autorenew> val) {
+    serverApproveAutorenewPollMessageHistoryId = DomainBase.getHistoryId(val);
+  }
+
   public Period getTransferPeriod() {
     return transferPeriod;
   }
@@ -125,16 +176,34 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     return serverApproveBillingEvent;
   }
 
+  @VisibleForTesting
+  @Nullable
+  public Long getServerApproveBillingEventHistoryId() {
+    return serverApproveBillingEventHistoryId;
+  }
+
   @Nullable
   public VKey<BillingEvent.Recurring> getServerApproveAutorenewEvent() {
     return serverApproveAutorenewEvent;
   }
 
+  @VisibleForTesting
+  @Nullable
+  public Long getServerApproveAutorenewEventHistoryId() {
+    return serverApproveAutorenewEventHistoryId;
+  }
+
   @Nullable
   public VKey<PollMessage.Autorenew> getServerApproveAutorenewPollMessage() {
     return serverApproveAutorenewPollMessage;
   }
 
+  @VisibleForTesting
+  @Nullable
+  public Long getServerApproveAutorenewPollMessageHistoryId() {
+    return serverApproveAutorenewPollMessageHistoryId;
+  }
+
   @Override
   public boolean isEmpty() {
     return EMPTY.equals(this);
@@ -168,18 +237,24 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     public Builder setServerApproveBillingEvent(
         VKey<BillingEvent.OneTime> serverApproveBillingEvent) {
       getInstance().serverApproveBillingEvent = serverApproveBillingEvent;
+      getInstance().serverApproveBillingEventHistoryId =
+          DomainBase.getHistoryId(serverApproveBillingEvent);
       return this;
     }
 
     public Builder setServerApproveAutorenewEvent(
         VKey<BillingEvent.Recurring> serverApproveAutorenewEvent) {
       getInstance().serverApproveAutorenewEvent = serverApproveAutorenewEvent;
+      getInstance().serverApproveAutorenewEventHistoryId =
+          DomainBase.getHistoryId(serverApproveAutorenewEvent);
       return this;
     }
 
     public Builder setServerApproveAutorenewPollMessage(
         VKey<PollMessage.Autorenew> serverApproveAutorenewPollMessage) {
       getInstance().serverApproveAutorenewPollMessage = serverApproveAutorenewPollMessage;
+      getInstance().serverApproveAutorenewPollMessageHistoryId =
+          DomainBase.getHistoryId(serverApproveAutorenewPollMessage);
       return this;
     }
   }

core/src/main/java/google/registry/module/ServletBase.java

@@ -22,7 +22,6 @@ import google.registry.request.RequestHandler;
 import google.registry.util.SystemClock;
 import java.io.IOException;
 import java.security.Security;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
@@ -51,13 +50,16 @@ public class ServletBase extends HttpServlet {
     // etc), we log the error but keep the main thread running. Also the shutdown hook will only be
     // registered if metric reporter starts up correctly.
     try {
-      metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS);
+      metricReporter.get().startAsync().awaitRunning(java.time.Duration.ofSeconds(10));
       logger.atInfo().log("Started up MetricReporter");
       LifecycleManager.getInstance()
           .setShutdownHook(
               () -> {
                 try {
-                  metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS);
+                  metricReporter
+                      .get()
+                      .stopAsync()
+                      .awaitTerminated(java.time.Duration.ofSeconds(10));
                   logger.atInfo().log("Shut down MetricReporter");
                 } catch (TimeoutException e) {
                   logger.atSevere().withCause(e).log("Failed to stop MetricReporter.");

core/src/main/java/google/registry/persistence/VKey.java

@@ -88,7 +88,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    */
   public static <T> VKey<T> create(Class<T> kind, long id) {
     checkArgument(
-        kind.isAssignableFrom(BackupGroupRoot.class),
+        BackupGroupRoot.class.isAssignableFrom(kind),
         "The kind %s is not a BackupGroupRoot and thus needs its entire entity group chain"
             + " specified in a parent",
         kind.getCanonicalName());
@@ -106,13 +106,90 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    */
   public static <T> VKey<T> create(Class<T> kind, String name) {
     checkArgument(
-        kind.isAssignableFrom(BackupGroupRoot.class),
+        BackupGroupRoot.class.isAssignableFrom(kind),
         "The kind %s is not a BackupGroupRoot and thus needs its entire entity group chain"
             + " specified in a parent",
         kind.getCanonicalName());
     return new VKey<T>(kind, Key.create(kind, name), name);
   }
 
+  /**
+   * Returns a clone with an ofy key restored from {@code ancestors}.
+   *
+   * <p>The arguments should generally consist of pairs of Class and value, where the Class is the
+   * kind of the ancestor key and the value is either a String or a Long.
+   *
+   * <p>For example, to restore the objectify key for
+   * DomainBase("COM-1234")/HistoryEntry(123)/PollEvent(567), one might use:
+   *
+   * <pre>{@code
+   * pollEvent.restoreOfy(DomainBase.class, "COM-1234", HistoryEntry.class, 567)
+   * }</pre>
+   *
+   * <p>The final key id or name is obtained from the SQL key. It is assumed that this value must be
+   * either a long integer or a {@code String} and that this proper identifier for the objectify
+   * key.
+   *
+   * <p>As a special case, an objectify Key may be used as the first ancestor instead of a Class,
+   * value pair.
+   */
+  public VKey<T> restoreOfy(Object... ancestors) {
+    Class lastClass = null;
+    Key<?> lastKey = null;
+    for (Object ancestor : ancestors) {
+      if (ancestor instanceof Class) {
+        if (lastClass != null) {
+          throw new IllegalArgumentException(ancestor + " used as a key value.");
+        }
+        lastClass = (Class) ancestor;
+        continue;
+      } else if (ancestor instanceof Key) {
+        if (lastKey != null) {
+          throw new IllegalArgumentException(
+              "Objectify keys may only be used for the first argument");
+        }
+        lastKey = (Key) ancestor;
+        continue;
+      }
+
+      // The argument should be a value.
+      if (lastClass == null) {
+        throw new IllegalArgumentException("Argument " + ancestor + " should be a class.");
+      }
+      if (ancestor instanceof Long) {
+        lastKey = Key.create(lastKey, lastClass, (Long) ancestor);
+      } else if (ancestor instanceof String) {
+        lastKey = Key.create(lastKey, lastClass, (String) ancestor);
+      } else {
+        throw new IllegalArgumentException("Key value " + ancestor + " must be a string or long.");
+      }
+      lastClass = null;
+    }
+
+    // Make sure we didn't end up with a dangling class with no value.
+    if (lastClass != null) {
+      throw new IllegalArgumentException("Missing value for last key of type " + lastClass);
+    }
+
+    Object sqlKey = getSqlKey();
+    Key<T> ofyKey =
+        sqlKey instanceof Long
+            ? Key.create(lastKey, getKind(), (Long) sqlKey)
+            : Key.create(lastKey, getKind(), (String) sqlKey);
+
+    return VKey.create((Class<T>) getKind(), sqlKey, ofyKey);
+  }
+
+  /**
+   * Returns a clone of {@code key} with an ofy key restored from {@code ancestors}.
+   *
+   * <p>This is the static form of the method restoreOfy() above. If {@code key} is null, it returns
+   * null.
+   */
+  public static <T> VKey<T> restoreOfyFrom(@Nullable VKey<T> key, Object... ancestors) {
+    return key == null ? null : key.restoreOfy(ancestors);
+  }
+
   /** Returns the type of the entity. */
   public Class<? extends T> getKind() {
     return this.kind;

core/src/main/java/google/registry/persistence/converter/CreateAutoTimestampConverter.java

@@ -32,7 +32,11 @@ public class CreateAutoTimestampConverter
     implements AttributeConverter<CreateAutoTimestamp, Timestamp> {
 
   @Override
-  public Timestamp convertToDatabaseColumn(CreateAutoTimestamp entity) {
+  @Nullable
+  public Timestamp convertToDatabaseColumn(@Nullable CreateAutoTimestamp entity) {
+    if (entity == null) {
+      return null;
+    }
     DateTime dateTime = firstNonNull(entity.getTimestamp(), jpaTm().getTransactionTime());
     return Timestamp.from(DateTimeUtils.toZonedDateTime(dateTime).toInstant());
   }

core/src/main/java/google/registry/persistence/converter/LocalDateConverter.java

@@ -14,17 +14,23 @@
 
 package google.registry.persistence.converter;
 
+import google.registry.util.DateTimeUtils;
+import java.sql.Date;
+import javax.persistence.AttributeConverter;
 import javax.persistence.Converter;
 import org.joda.time.LocalDate;
-import org.joda.time.format.ISODateTimeFormat;
 
-/** JPA converter for {@link LocalDate}. */
+/** JPA converter for {@link LocalDate}, to/from {@link Date}. */
 @Converter(autoApply = true)
-public class LocalDateConverter extends ToStringConverterBase<LocalDate> {
+public class LocalDateConverter implements AttributeConverter<LocalDate, Date> {
 
-  /** Converts the string (a date in ISO-8601 format) into a LocalDate. */
   @Override
-  public LocalDate convertToEntityAttribute(String columnValue) {
-    return (columnValue == null) ? null : LocalDate.parse(columnValue, ISODateTimeFormat.date());
+  public Date convertToDatabaseColumn(LocalDate attribute) {
+    return attribute == null ? null : DateTimeUtils.toSqlDate(attribute);
+  }
+
+  @Override
+  public LocalDate convertToEntityAttribute(Date dbData) {
+    return dbData == null ? null : DateTimeUtils.toLocalDate(dbData);
   }
 }

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -15,6 +15,7 @@
 package google.registry.persistence.transaction;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
@@ -25,6 +26,7 @@ import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig;
 import google.registry.persistence.JpaRetries;
@@ -224,7 +226,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void saveNew(Object entity) {
+  public void insert(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
     assertInTransaction();
     getEntityManager().persist(entity);
@@ -232,14 +234,24 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void saveAllNew(ImmutableCollection<?> entities) {
+  public void insertAll(ImmutableCollection<?> entities) {
     checkArgumentNotNull(entities, "entities must be specified");
     assertInTransaction();
-    entities.forEach(this::saveNew);
+    entities.forEach(this::insert);
   }
 
   @Override
-  public void saveNewOrUpdate(Object entity) {
+  public void insertWithoutBackup(Object entity) {
+    insert(entity);
+  }
+
+  @Override
+  public void insertAllWithoutBackup(ImmutableCollection<?> entities) {
+    insertAll(entities);
+  }
+
+  @Override
+  public void put(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
     assertInTransaction();
     getEntityManager().merge(entity);
@@ -247,17 +259,27 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void saveNewOrUpdateAll(ImmutableCollection<?> entities) {
+  public void putAll(ImmutableCollection<?> entities) {
     checkArgumentNotNull(entities, "entities must be specified");
     assertInTransaction();
-    entities.forEach(this::saveNewOrUpdate);
+    entities.forEach(this::put);
+  }
+
+  @Override
+  public void putWithoutBackup(Object entity) {
+    put(entity);
+  }
+
+  @Override
+  public void putAllWithoutBackup(ImmutableCollection<?> entities) {
+    putAll(entities);
   }
 
   @Override
   public void update(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
     assertInTransaction();
-    checkArgument(checkExists(entity), "Given entity does not exist");
+    checkArgument(exists(entity), "Given entity does not exist");
     getEntityManager().merge(entity);
     transactionInfo.get().addUpdate(entity);
   }
@@ -270,22 +292,32 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public <T> boolean checkExists(VKey<T> key) {
-    checkArgumentNotNull(key, "key must be specified");
-    EntityType<?> entityType = getEntityType(key.getKind());
-    ImmutableSet<EntityId> entityIds = getEntityIdsFromSqlKey(entityType, key.getSqlKey());
-    return checkExists(entityType.getName(), entityIds);
+  public void updateWithoutBackup(Object entity) {
+    update(entity);
   }
 
   @Override
-  public boolean checkExists(Object entity) {
+  public void updateAllWithoutBackup(ImmutableCollection<?> entities) {
+    updateAll(entities);
+  }
+
+  @Override
+  public <T> boolean exists(VKey<T> key) {
+    checkArgumentNotNull(key, "key must be specified");
+    EntityType<?> entityType = getEntityType(key.getKind());
+    ImmutableSet<EntityId> entityIds = getEntityIdsFromSqlKey(entityType, key.getSqlKey());
+    return exists(entityType.getName(), entityIds);
+  }
+
+  @Override
+  public boolean exists(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
     EntityType<?> entityType = getEntityType(entity.getClass());
     ImmutableSet<EntityId> entityIds = getEntityIdsFromEntity(entityType, entity);
-    return checkExists(entityType.getName(), entityIds);
+    return exists(entityType.getName(), entityIds);
   }
 
-  private boolean checkExists(String entityName, ImmutableSet<EntityId> entityIds) {
+  private boolean exists(String entityName, ImmutableSet<EntityId> entityIds) {
     assertInTransaction();
     TypedQuery<Integer> query =
         getEntityManager()
@@ -315,6 +347,14 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     return result;
   }
 
+  @Override
+  public <T> T load(T entity) {
+    checkArgumentNotNull(entity, "entity must be specified");
+    assertInTransaction();
+    return (T)
+        load(VKey.createSql(entity.getClass(), emf.getPersistenceUnitUtil().getIdentifier(entity)));
+  }
+
   @Override
   public <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys) {
     checkArgumentNotNull(keys, "keys must be specified");
@@ -342,6 +382,11 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
             .getResultList());
   }
 
+  @Override
+  public <T> ImmutableList<T> loadAll(Iterable<T> entities) {
+    return Streams.stream(entities).map(this::load).collect(toImmutableList());
+  }
+
   private int internalDelete(VKey<?> key) {
     checkArgumentNotNull(key, "key must be specified");
     assertInTransaction();
@@ -366,6 +411,37 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     vKeys.forEach(this::internalDelete);
   }
 
+  @Override
+  public void delete(Object entity) {
+    checkArgumentNotNull(entity, "entity must be specified");
+    assertInTransaction();
+    Object managedEntity = entity;
+    if (!getEntityManager().contains(entity)) {
+      managedEntity = getEntityManager().merge(entity);
+    }
+    getEntityManager().remove(managedEntity);
+  }
+
+  @Override
+  public void deleteWithoutBackup(VKey<?> key) {
+    delete(key);
+  }
+
+  @Override
+  public void deleteWithoutBackup(Iterable<? extends VKey<?>> keys) {
+    delete(keys);
+  }
+
+  @Override
+  public void deleteWithoutBackup(Object entity) {
+    delete(entity);
+  }
+
+  @Override
+  public void clearSessionCache() {
+    // This is an intended no-op method as there is no session cache in Postgresql.
+  }
+
   @Override
   public <T> void assertDelete(VKey<T> key) {
     if (internalDelete(key) != 1) {

core/src/main/java/google/registry/persistence/transaction/Transaction.java

@@ -208,7 +208,7 @@ public class Transaction extends ImmutableObject implements Buildable {
 
     @Override
     public void writeToDatastore() {
-      ofyTm().saveNewOrUpdate(entity);
+      ofyTm().put(entity);
     }
 
     @Override

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -86,16 +86,64 @@ public interface TransactionManager {
   DateTime getTransactionTime();
 
   /** Persists a new entity in the database, throws exception if the entity already exists. */
-  void saveNew(Object entity);
+  void insert(Object entity);
 
   /** Persists all new entities in the database, throws exception if any entity already exists. */
-  void saveAllNew(ImmutableCollection<?> entities);
+  void insertAll(ImmutableCollection<?> entities);
+
+  /**
+   * Persists a new entity in the database without writing any backup if the underlying database is
+   * Datastore.
+   *
+   * <p>This method is for the sake of keeping a single code path when replacing ofy() with tm() in
+   * the application code. When the method is invoked with Datastore, it won't write the commit log
+   * backup; when invoked with Cloud SQL, it behaves the same as the method which doesn't have
+   * "WithoutBackup" in its method name because it is not necessary to have the backup mechanism in
+   * SQL.
+   */
+  void insertWithoutBackup(Object entity);
+
+  /**
+   * Persists all new entities in the database without writing any backup if the underlying database
+   * is Datastore.
+   *
+   * <p>This method is for the sake of keeping a single code path when replacing ofy() with tm() in
+   * the application code. When the method is invoked with Datastore, it won't write the commit log
+   * backup; when invoked with Cloud SQL, it behaves the same as the method which doesn't have
+   * "WithoutBackup" in its method name because it is not necessary to have the backup mechanism in
+   * SQL.
+   */
+  void insertAllWithoutBackup(ImmutableCollection<?> entities);
 
   /** Persists a new entity or update the existing entity in the database. */
-  void saveNewOrUpdate(Object entity);
+  void put(Object entity);
 
   /** Persists all new entities or update the existing entities in the database. */
-  void saveNewOrUpdateAll(ImmutableCollection<?> entities);
+  void putAll(ImmutableCollection<?> entities);
+
+  /**
+   * Persists a new entity or update the existing entity in the database without writing any backup
+   * if the underlying database is Datastore.
+   *
+   * <p>This method is for the sake of keeping a single code path when replacing ofy() with tm() in
+   * the application code. When the method is invoked with Datastore, it won't write the commit log
+   * backup; when invoked with Cloud SQL, it behaves the same as the method which doesn't have
+   * "WithoutBackup" in its method name because it is not necessary to have the backup mechanism in
+   * SQL.
+   */
+  void putWithoutBackup(Object entity);
+
+  /**
+   * Persists all new entities or update the existing entities in the database without writing any
+   * backup if the underlying database is Datastore.
+   *
+   * <p>This method is for the sake of keeping a single code path when replacing ofy() with tm() in
+   * the application code. When the method is invoked with Datastore, it won't write the commit log
+   * backup; when invoked with Cloud SQL, it behaves the same as the method which doesn't have
+   * "WithoutBackup" in its method name because it is not necessary to have the backup mechanism in
+   * SQL.
+   */
+  void putAllWithoutBackup(ImmutableCollection<?> entities);
 
   /** Updates an entity in the database, throws exception if the entity does not exist. */
   void update(Object entity);
@@ -103,11 +151,35 @@ public interface TransactionManager {
   /** Updates all entities in the database, throws exception if any entity does not exist. */
   void updateAll(ImmutableCollection<?> entities);
 
+  /**
+   * Updates an entity in the database without writing any backup if the underlying database is
+   * Datastore.
+   *
+   * <p>This method is for the sake of keeping a single code path when replacing ofy() with tm() in
+   * the application code. When the method is invoked with Datastore, it won't write the commit log
+   * backup; when invoked with Cloud SQL, it behaves the same as the method which doesn't have
+   * "WithoutBackup" in its method name because it is not necessary to have the backup mechanism in
+   * SQL.
+   */
+  void updateWithoutBackup(Object entity);
+
+  /**
+   * Updates all entities in the database without writing any backup if the underlying database is
+   * Datastore.
+   *
+   * <p>This method is for the sake of keeping a single code path when replacing ofy() with tm() in
+   * the application code. When the method is invoked with Datastore, it won't write the commit log
+   * backup; when invoked with Cloud SQL, it behaves the same as the method which doesn't have
+   * "WithoutBackup" in its method name because it is not necessary to have the backup mechanism in
+   * SQL.
+   */
+  void updateAllWithoutBackup(ImmutableCollection<?> entities);
+
   /** Returns whether the given entity with same ID exists. */
-  boolean checkExists(Object entity);
+  boolean exists(Object entity);
 
   /** Returns whether the entity of given key exists. */
-  <T> boolean checkExists(VKey<T> key);
+  <T> boolean exists(VKey<T> key);
 
   /** Loads the entity by its id, returns empty if the entity doesn't exist. */
   <T> Optional<T> maybeLoad(VKey<T> key);
@@ -115,6 +187,11 @@ public interface TransactionManager {
   /** Loads the entity by its id, throws NoSuchElementException if it doesn't exist. */
   <T> T load(VKey<T> key);
 
+  /**
+   * Loads the given entity from the database, throws NoSuchElementException if it doesn't exist.
+   */
+  <T> T load(T entity);
+
   /**
    * Loads the set of entities by their key id.
    *
@@ -125,9 +202,38 @@ public interface TransactionManager {
   /** Loads all entities of the given type, returns empty if there is no such entity. */
   <T> ImmutableList<T> loadAll(Class<T> clazz);
 
+  /**
+   * Loads all given entities from the database, throws NoSuchElementException if it doesn't exist.
+   */
+  <T> ImmutableList<T> loadAll(Iterable<T> entities);
+
   /** Deletes the entity by its id. */
   void delete(VKey<?> key);
 
   /** Deletes the set of entities by their key id. */
   void delete(Iterable<? extends VKey<?>> keys);
+
+  /** Deletes the given entity from the database. */
+  void delete(Object entity);
+
+  /**
+   * Deletes the entity by its id without writing any backup if the underlying database is
+   * Datastore.
+   */
+  void deleteWithoutBackup(VKey<?> key);
+
+  /**
+   * Deletes the set of entities by their key id without writing any backup if the underlying
+   * database is Datastore.
+   */
+  void deleteWithoutBackup(Iterable<? extends VKey<?>> keys);
+
+  /**
+   * Deletes the given entity from the database without writing any backup if the underlying
+   * database is Datastore.
+   */
+  void deleteWithoutBackup(Object entity);
+
+  /** Clears the session cache if the underlying database is Datastore, otherwise it is a no-op. */
+  void clearSessionCache();
 }

core/src/main/java/google/registry/persistence/transaction/TransactionManagerUtil.java

@@ -0,0 +1,110 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.transaction;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import google.registry.model.ofy.DatastoreTransactionManager;
+import java.util.function.Supplier;
+
+/** Utility class that provides supplementary methods for {@link TransactionManager}. */
+public class TransactionManagerUtil {
+
+  /**
+   * Returns the result of the given {@link Supplier}.
+   *
+   * <p>If {@link TransactionManagerFactory#tm()} returns a {@link JpaTransactionManager} instance,
+   * the {@link Supplier} is executed in a transaction.
+   */
+  public static <T> T transactIfJpaTm(Supplier<T> supplier) {
+    if (tm() instanceof JpaTransactionManager) {
+      return tm().transact(supplier);
+    } else {
+      return supplier.get();
+    }
+  }
+
+  /**
+   * Executes the given {@link Runnable}.
+   *
+   * <p>If {@link TransactionManagerFactory#tm()} returns a {@link JpaTransactionManager} instance,
+   * the {@link Runnable} is executed in a transaction.
+   */
+  public static void transactIfJpaTm(Runnable runnable) {
+    transactIfJpaTm(
+        () -> {
+          runnable.run();
+          return null;
+        });
+  }
+
+  /**
+   * Executes either {@code ofyRunnable} if {@link TransactionManagerFactory#tm()} returns a {@link
+   * JpaTransactionManager} instance, or {@code jpaRunnable} if {@link
+   * TransactionManagerFactory#tm()} returns a {@link DatastoreTransactionManager} instance.
+   */
+  public static void ofyOrJpaTm(Runnable ofyRunnable, Runnable jpaRunnable) {
+    ofyOrJpaTm(
+        () -> {
+          ofyRunnable.run();
+          return null;
+        },
+        () -> {
+          jpaRunnable.run();
+          return null;
+        });
+  }
+
+  /**
+   * Returns the result from either {@code ofySupplier} if {@link TransactionManagerFactory#tm()}
+   * returns a {@link JpaTransactionManager} instance, or {@code jpaSupplier} if {@link
+   * TransactionManagerFactory#tm()} returns a {@link DatastoreTransactionManager} instance.
+   */
+  public static <T> T ofyOrJpaTm(Supplier<T> ofySupplier, Supplier<T> jpaSupplier) {
+    if (tm() instanceof DatastoreTransactionManager) {
+      return ofySupplier.get();
+    } else if (tm() instanceof JpaTransactionManager) {
+      return jpaSupplier.get();
+    } else {
+      throw new IllegalStateException(
+          "Expected tm() to be DatastoreTransactionManager or JpaTransactionManager, but got "
+              + tm().getClass());
+    }
+  }
+
+  /**
+   * Executes the given {@link Runnable} if {@link TransactionManagerFactory#tm()} returns a {@link
+   * DatastoreTransactionManager} instance, otherwise does nothing.
+   */
+  public static void ofyTmOrDoNothing(Runnable ofyRunnable) {
+    if (tm() instanceof DatastoreTransactionManager) {
+      ofyRunnable.run();
+    }
+  }
+
+  /**
+   * Returns the result from the given {@link Supplier} if {@link TransactionManagerFactory#tm()}
+   * returns a {@link DatastoreTransactionManager} instance, otherwise returns null.
+   */
+  public static <T> T ofyTmOrDoNothing(Supplier<T> ofySupplier) {
+    if (tm() instanceof DatastoreTransactionManager) {
+      return ofySupplier.get();
+    } else {
+      return null;
+    }
+  }
+
+  private TransactionManagerUtil() {}
+}

core/src/main/java/google/registry/schema/tld/PremiumListCache.java

@@ -18,7 +18,6 @@ import static google.registry.config.RegistryConfig.getDomainLabelListCacheDurat
 import static google.registry.config.RegistryConfig.getSingletonCachePersistDuration;
 import static google.registry.config.RegistryConfig.getStaticPremiumListMaxCachedEntries;
 import static google.registry.schema.tld.PremiumListDao.getPriceForLabel;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
@@ -48,7 +47,7 @@ class PremiumListCache {
   static LoadingCache<String, Optional<PremiumList>> createCachePremiumLists(
       Duration cachePersistDuration) {
     return CacheBuilder.newBuilder()
-        .expireAfterWrite(cachePersistDuration.getMillis(), MILLISECONDS)
+        .expireAfterWrite(java.time.Duration.ofMillis(cachePersistDuration.getMillis()))
         .build(
             new CacheLoader<String, Optional<PremiumList>>() {
               @Override
@@ -81,7 +80,7 @@ class PremiumListCache {
   static LoadingCache<RevisionIdAndLabel, Optional<BigDecimal>> createCachePremiumEntries(
       Duration cachePersistDuration) {
     return CacheBuilder.newBuilder()
-        .expireAfterWrite(cachePersistDuration.getMillis(), MILLISECONDS)
+        .expireAfterWrite(java.time.Duration.ofMillis(cachePersistDuration.getMillis()))
         .maximumSize(getStaticPremiumListMaxCachedEntries())
         .build(
             new CacheLoader<RevisionIdAndLabel, Optional<BigDecimal>>() {

core/src/main/java/google/registry/tmch/TmchCertificateAuthority.java

@@ -18,7 +18,6 @@ import static google.registry.config.RegistryConfig.ConfigModule.TmchCaMode.PILO
 import static google.registry.config.RegistryConfig.ConfigModule.TmchCaMode.PRODUCTION;
 import static google.registry.config.RegistryConfig.getSingletonCacheRefreshDuration;
 import static google.registry.util.ResourceUtils.readResourceUtf8;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
@@ -77,7 +76,8 @@ public final class TmchCertificateAuthority {
    */
   private static final LoadingCache<TmchCaMode, X509CRL> CRL_CACHE =
       CacheBuilder.newBuilder()
-          .expireAfterWrite(getSingletonCacheRefreshDuration().getMillis(), MILLISECONDS)
+          .expireAfterWrite(
+              java.time.Duration.ofMillis(getSingletonCacheRefreshDuration().getMillis()))
           .build(
               new CacheLoader<TmchCaMode, X509CRL>() {
                 @Override

core/src/main/java/google/registry/tools/CompareDbBackups.java

@@ -18,7 +18,6 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Sets.SetView;
 import java.io.File;
-import java.util.function.Predicate;
 
 /**
  * Compares two Datastore backups in V3 format on local file system. This is for use in tests and
@@ -30,8 +29,10 @@ import java.util.function.Predicate;
  */
 class CompareDbBackups {
   private static final String DS_V3_BACKUP_FILE_PREFIX = "output-";
-  private static final Predicate<File> DATA_FILE_MATCHER =
-      file -> file.isFile() && file.getName().startsWith(DS_V3_BACKUP_FILE_PREFIX);
+
+  private static boolean isDatastoreV3File(File file) {
+    return file.isFile() && file.getName().startsWith(DS_V3_BACKUP_FILE_PREFIX);
+  }
 
   public static void main(String[] args) {
     if (args.length != 2) {
@@ -40,9 +41,11 @@ class CompareDbBackups {
     }
 
     ImmutableSet<EntityWrapper> entities1 =
-        RecordAccumulator.readDirectory(new File(args[0]), DATA_FILE_MATCHER).getEntityWrapperSet();
+        RecordAccumulator.readDirectory(new File(args[0]), CompareDbBackups::isDatastoreV3File)
+            .getEntityWrapperSet();
     ImmutableSet<EntityWrapper> entities2 =
-        RecordAccumulator.readDirectory(new File(args[1]), DATA_FILE_MATCHER).getEntityWrapperSet();
+        RecordAccumulator.readDirectory(new File(args[1]), CompareDbBackups::isDatastoreV3File)
+            .getEntityWrapperSet();
 
     // Calculate the entities added and removed.
     SetView<EntityWrapper> added = Sets.difference(entities2, entities1);

core/src/main/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java

@@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import com.google.common.flogger.FluentLogger;
+import google.registry.flows.certs.CertificateChecker;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarAddress;
 import google.registry.model.registry.Registry;
@@ -49,6 +50,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
+import javax.inject.Inject;
 import org.joda.money.CurrencyUnit;
 import org.joda.time.DateTime;
 
@@ -57,9 +59,9 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
 
   static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  @Parameter(
-      description = "Client identifier of the registrar account",
-      required = true)
+  @Inject CertificateChecker certificateChecker;
+
+  @Parameter(description = "Client identifier of the registrar account", required = true)
   List<String> mainParameters;
 
   @Parameter(
@@ -356,11 +358,21 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
       }
       if (clientCertificateFilename != null) {
         String asciiCert = new String(Files.readAllBytes(clientCertificateFilename), US_ASCII);
+        // An empty certificate file is allowed in order to provide a functionality for removing an
+        // existing certificate without providing a replacement. An uploaded empty certificate file
+        // will prevent the registrar from being able to establish EPP connections.
+        if (!asciiCert.equals("")) {
+          certificateChecker.validateCertificate(asciiCert);
+        }
         builder.setClientCertificate(asciiCert, now);
       }
+
       if (failoverClientCertificateFilename != null) {
         String asciiCert =
             new String(Files.readAllBytes(failoverClientCertificateFilename), US_ASCII);
+        if (!asciiCert.equals("")) {
+          certificateChecker.validateCertificate(asciiCert);
+        }
         builder.setFailoverClientCertificate(asciiCert, now);
       }
       if (!isNullOrEmpty(clientCertificateHash)) {

core/src/main/java/google/registry/tools/CreateRegistrarCommand.java

@@ -72,7 +72,7 @@ final class CreateRegistrarCommand extends CreateOrUpdateRegistrarCommand
 
   @Override
   void saveToCloudSql(Registrar registrar) {
-    jpaTm().saveNew(registrar);
+    jpaTm().insert(registrar);
   }
 
   @Nullable

core/src/main/java/google/registry/tools/DedupeEntityIdsCommand.java

@@ -19,42 +19,22 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
 import com.google.appengine.api.datastore.KeyFactory;
 import com.google.common.base.Splitter;
 import com.google.common.io.CharStreams;
 import com.google.common.io.Files;
 import com.googlecode.objectify.Key;
 import google.registry.model.ImmutableObject;
-import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.DomainBase;
 import google.registry.util.NonFinalForTesting;
+import google.registry.util.TypeUtils.TypeInstantiator;
 import java.io.File;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.List;
 
-/**
- * Command to resave entities with a unique id.
- *
- * <p>This command is used to address the duplicate id issue we found for certain {@link
- * BillingEvent.OneTime} entities. The command reassigns an application wide unique id to the
- * problematic entity and resaves it, it also resaves the entity having reference to the problematic
- * entity with the updated id.
- *
- * <p>To use this command, you will need to provide the path to a file containing a list of strings
- * representing the literal of Objectify key for the problematic entities. An example key literal
- * is:
- *
- * <pre>
- * "DomainBase", "111111-TEST", "HistoryEntry", 2222222, "OneTime", 3333333
- * </pre>
- *
- * <p>Note that the double quotes are part of the key literal. The key literal can be retrieved from
- * the column <code>__key__.path</code> in BigQuery.
- */
-@Parameters(separators = " =", commandDescription = "Resave entities with a unique id.")
-public class ResaveEntitiesWithUniqueIdCommand extends MutatingCommand {
+/** Base Command to dedupe entities with duplicate IDs. */
+abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
 
   @Parameter(
       names = "--key_paths_file",
@@ -66,7 +46,9 @@ public class ResaveEntitiesWithUniqueIdCommand extends MutatingCommand {
 
   @NonFinalForTesting private static InputStream stdin = System.in;
 
-  private String keyChangeMessage;
+  private StringBuilder changeMessage = new StringBuilder();
+
+  abstract void dedupe(T entity);
 
   @Override
   protected void init() throws Exception {
@@ -85,8 +67,9 @@ public class ResaveEntitiesWithUniqueIdCommand extends MutatingCommand {
                 keyPathsFile == null ? "STDIN" : "File " + keyPathsFile.getAbsolutePath()));
         continue;
       }
-      if (entity instanceof BillingEvent.OneTime) {
-        resaveBillingEvent((BillingEvent.OneTime) entity);
+      Class<T> clazz = new TypeInstantiator<T>(getClass()) {}.getExactType();
+      if (clazz.isInstance(entity)) {
+        dedupe((T) entity);
       } else {
         throw new IllegalArgumentException("Unsupported entity key: " + untypedKey);
       }
@@ -96,37 +79,19 @@ public class ResaveEntitiesWithUniqueIdCommand extends MutatingCommand {
 
   @Override
   protected void postBatchExecute() {
-    System.out.println(keyChangeMessage);
+    System.out.println(changeMessage);
   }
 
-  private void deleteOldAndSaveNewEntity(ImmutableObject oldEntity, ImmutableObject newEntity) {
+  void stageEntityKeyChange(ImmutableObject oldEntity, ImmutableObject newEntity) {
     stageEntityChange(oldEntity, null);
     stageEntityChange(null, newEntity);
+    appendChangeMessage(
+        String.format(
+            "Changed entity key from: %s to: %s", Key.create(oldEntity), Key.create(newEntity)));
   }
 
-  private void resaveBillingEvent(BillingEvent.OneTime billingEvent) {
-    Key<BillingEvent> key = Key.create(billingEvent);
-    Key<DomainBase> domainKey = getGrandParentAsDomain(key);
-    DomainBase domain = ofy().load().key(domainKey).now();
-
-    // The BillingEvent.OneTime entity to be resaved should be the billing event created a few
-    // years ago, so they should not be referenced from TransferData and GracePeriod in the domain.
-    assertNotInDomainTransferData(domain, key);
-    domain
-        .getGracePeriods()
-        .forEach(
-            gracePeriod ->
-                checkState(
-                    !gracePeriod.getOneTimeBillingEvent().getOfyKey().equals(key),
-                    "Entity %s is referenced by a grace period in domain %s",
-                    key,
-                    domainKey));
-
-    // By setting id to 0L, Buildable.build() will assign an application wide unique id to it.
-    BillingEvent.OneTime uniqIdBillingEvent = billingEvent.asBuilder().setId(0L).build();
-    deleteOldAndSaveNewEntity(billingEvent, uniqIdBillingEvent);
-    keyChangeMessage =
-        String.format("Old Entity Key: %s New Entity Key: %s", key, Key.create(uniqIdBillingEvent));
+  void appendChangeMessage(String message) {
+    changeMessage.append(message);
   }
 
   private static boolean isKind(Key<?> key, Class<?> clazz) {
@@ -165,7 +130,7 @@ public class ResaveEntitiesWithUniqueIdCommand extends MutatingCommand {
     return literal.substring(1, literal.length() - 1);
   }
 
-  private static Key<DomainBase> getGrandParentAsDomain(Key<?> key) {
+  static Key<DomainBase> getGrandParentAsDomain(Key<?> key) {
     Key<?> grandParent;
     try {
       grandParent = key.getParent().getParent();
@@ -178,19 +143,4 @@ public class ResaveEntitiesWithUniqueIdCommand extends MutatingCommand {
     }
     return (Key<DomainBase>) grandParent;
   }
-
-  private static void assertNotInDomainTransferData(DomainBase domainBase, Key<?> key) {
-    if (!domainBase.getTransferData().isEmpty()) {
-      domainBase
-          .getTransferData()
-          .getServerApproveEntities()
-          .forEach(
-              entityKey ->
-                  checkState(
-                      !entityKey.getOfyKey().equals(key),
-                      "Entity %s is referenced by the transfer data in domain %s",
-                      key,
-                      domainBase.createVKey().getOfyKey()));
-    }
-  }
 }

core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java

@@ -0,0 +1,88 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+
+import com.beust.jcommander.Parameters;
+import com.googlecode.objectify.Key;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingEvent.OneTime;
+import google.registry.model.domain.DomainBase;
+
+/**
+ * Command to dedupe {@link BillingEvent.OneTime} entities having duplicate IDs.
+ *
+ * <p>This command is used to address the duplicate id issue we found for certain {@link
+ * BillingEvent.OneTime} entities. The command reassigns an application wide unique id to the
+ * problematic entity and resaves it, it also resaves the entity having reference to the problematic
+ * entity with the updated id.
+ *
+ * <p>To use this command, you will need to provide the path to a file containing a list of strings
+ * representing the literal of Objectify key for the problematic entities. An example key literal
+ * is:
+ *
+ * <pre>
+ * "DomainBase", "111111-TEST", "HistoryEntry", 2222222, "OneTime", 3333333
+ * </pre>
+ *
+ * <p>Note that the double quotes are part of the key literal. The key literal can be retrieved from
+ * the column <code>__key__.path</code> in BigQuery.
+ */
+@Parameters(
+    separators = " =",
+    commandDescription = "Dedupe BillingEvent.OneTime entities with duplicate IDs.")
+public class DedupeOneTimeBillingEventIdsCommand extends DedupeEntityIdsCommand<OneTime> {
+
+  @Override
+  void dedupe(OneTime entity) {
+    Key<BillingEvent> key = Key.create(entity);
+    Key<DomainBase> domainKey = getGrandParentAsDomain(key);
+    DomainBase domain = ofy().load().key(domainKey).now();
+
+    // The BillingEvent.OneTime entity to be resaved should be the billing event created a few
+    // years ago, so they should not be referenced from TransferData and GracePeriod in the domain.
+    assertNotInDomainTransferData(domain, key);
+    domain
+        .getGracePeriods()
+        .forEach(
+            gracePeriod ->
+                checkState(
+                    !gracePeriod.getOneTimeBillingEvent().getOfyKey().equals(key),
+                    "Entity %s is referenced by a grace period in domain %s",
+                    key,
+                    domainKey));
+
+    // By setting id to 0L, Buildable.build() will assign an application wide unique id to it.
+    BillingEvent.OneTime uniqIdBillingEvent = entity.asBuilder().setId(0L).build();
+    stageEntityKeyChange(entity, uniqIdBillingEvent);
+  }
+
+  private static void assertNotInDomainTransferData(DomainBase domainBase, Key<?> key) {
+    if (!domainBase.getTransferData().isEmpty()) {
+      domainBase
+          .getTransferData()
+          .getServerApproveEntities()
+          .forEach(
+              entityKey ->
+                  checkState(
+                      !entityKey.getOfyKey().equals(key),
+                      "Entity %s is referenced by the transfer data in domain %s",
+                      key,
+                      domainBase.createVKey().getOfyKey()));
+    }
+  }
+}