Add a SQL schema and DAO for KmsSecretRevision (#840)

* Add a SQL schema and DAO for KmsSecretRevision

The dual-object nature of KmsSecret and KmsSecretRevision will not be
necessary once we have moved to SQL. In that world, the only object will
be the one now called KmsSecretRevision. KmsSecretRevision already
stores its parent so all we need to do is convert that key to the String
secretName (or from the secretName to the key, if loading from SQL) and
select the max revision ID for a given secret name.

In a future PR, we will add a dual-writing DAO to these objects and
perform the dual writes, similar to how ReservedList functions.

* Regenerate diagram

* Rename revisionId and cryptoKeyVersionName

* Fix SQL files and diagram

build.gradle

@@ -191,7 +191,7 @@ allprojects {
 }
 
 task runPresubmits(type: Exec) {
-  executable '/usr/bin/python'
+  executable '/usr/bin/python3'
   args('config/presubmits.py')
 }
 
@@ -260,7 +260,7 @@ subprojects {
       // in the 'configurations' block, the following code must run after
       // project evaluation, when all configurations have been created.
       configurations.each {
-        if (it.name != 'dependencyLicenseReport') {
+        if (it.name != 'dependencyLicenseReport' && it.name != 'integration') {
           it.resolutionStrategy.activateDependencyLocking()
         }
       }

buildSrc/gradle/dependency-locks/annotationProcessor.lockfile

@@ -1,25 +1,28 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto.value:auto-value:1.6.3
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

buildSrc/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/gradle/dependency-locks/annotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/gradle/dependency-locks/errorprone.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/gradle/dependency-locks/testingAnnotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

common/src/main/java/google/registry/util/DateTimeUtils.java

@@ -24,6 +24,7 @@ import java.time.ZonedDateTime;
 import java.util.TimeZone;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.LocalDate;
 
 /** Utilities methods and constants related to Joda {@link DateTime} objects. */
 public class DateTimeUtils {
@@ -108,4 +109,12 @@ public class DateTimeUtils {
         zonedDateTime.toInstant().toEpochMilli(),
         DateTimeZone.forTimeZone(TimeZone.getTimeZone(zonedDateTime.getZone())));
   }
+
+  public static java.sql.Date toSqlDate(LocalDate localDate) {
+    return new java.sql.Date(localDate.toDateTimeAtStartOfDay().getMillis());
+  }
+
+  public static LocalDate toLocalDate(java.sql.Date date) {
+    return new LocalDate(date.getTime(), DateTimeZone.UTC);
+  }
 }

common/src/main/java/google/registry/util/SystemSleeper.java

@@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.util.concurrent.Uninterruptibles;
 import java.io.Serializable;
-import java.util.concurrent.TimeUnit;
 import javax.annotation.concurrent.ThreadSafe;
 import javax.inject.Inject;
 import org.joda.time.ReadableDuration;
@@ -41,6 +40,6 @@ public final class SystemSleeper implements Sleeper, Serializable {
   @Override
   public void sleepUninterruptibly(ReadableDuration duration) {
     checkArgument(duration.getMillis() >= 0);
-    Uninterruptibles.sleepUninterruptibly(duration.getMillis(), TimeUnit.MILLISECONDS);
+    Uninterruptibles.sleepUninterruptibly(java.time.Duration.ofMillis(duration.getMillis()));
   }
 }

config/nom_build.py

@@ -16,7 +16,7 @@
 """
 
 import argparse
-import attr
+import dataclasses
 import io
 import os
 import shutil
@@ -25,7 +25,7 @@ import sys
 from typing import List, Union
 
 
-@attr.s(auto_attribs=True)
+@dataclasses.dataclass
 class Property:
     name : str = ''
     desc : str = ''
@@ -39,7 +39,7 @@ class Property:
                 raise ValidationError('value of {self.name} must be "true" or '
                                       '"false".')
 
-@attr.s(auto_attribs=True)
+@dataclasses.dataclass
 class GradleFlag:
     flags : Union[str, List[str]]
     desc : str
@@ -75,6 +75,7 @@ Pseudo-commands:
 """
 
 # Define all of our special gradle properties here.
+# TODO(b/169318491): use consistent naming style for properties and variables.
 PROPERTIES = [
     Property('mavenUrl',
              'URL to use for the main maven repository (defaults to maven '
@@ -124,6 +125,9 @@ PROPERTIES = [
              'server/schema integration tests. Please refer to <a '
              'href="./integration/README.md">integration project</a> for more '
              'information.'),
+    Property('baseSchemaTag',
+             'The nomulus version tag of the schema for use in the schema'
+             'deployment integration test (:db:schemaIncrementalDeployTest)'),
     Property('schema_version',
              'The nomulus version tag of the schema for use in a database'
              'integration test.'),

config/presubmits.py

@@ -18,6 +18,7 @@ Error Prone) so we must write them manually.
 """
 
 import os
+from typing import List, Tuple
 import sys
 import re
 
@@ -178,6 +179,90 @@ PRESUBMITS = {
         "JavaScript files should not include console logging."
 }
 
+# Note that this regex only works for one kind of Flyway file.  If we want to
+# start using "R" and "U" files we'll need to update this script.
+FLYWAY_FILE_RX = re.compile(r'V(\d+)__.*')
+
+
+def get_seqnum(filename: str, location: str) -> int:
+  """Extracts the sequence number from a filename."""
+  m = FLYWAY_FILE_RX.match(filename)
+  if m is None:
+    raise ValueError('Illegal Flyway filename: %s in %s' % (filename, location))
+  return int(m.group(1))
+
+
+def files_by_seqnum(files: List[str], location: str) -> List[Tuple[int, str]]:
+  """Returns the list of seqnum, filename sorted by sequence number."""
+  return [(get_seqnum(filename, location), filename) for filename in files]
+
+
+def has_valid_order(indexed_files: List[Tuple[int, str]], location: str) -> bool:
+  """Verify that sequence numbers are in order without gaps or duplicates.
+
+  Args:
+    files: List of seqnum, filename for a list of Flyway files.
+    location: Where the list of files came from (for error reporting).
+
+  Returns:
+    True if the file list is valid.
+  """
+  last_index = 0
+  valid = True
+  for seqnum, filename in indexed_files:
+    if seqnum == last_index:
+      print('duplicate Flyway file sequence number found in %s: %s' %
+            (location, filename))
+      valid = False
+    elif seqnum < last_index:
+      print('File %s in %s is out of order.' % (filename, location))
+      valid = False
+    elif seqnum != last_index + 1:
+      print('Missing Flyway sequence number %d in %s.  Next file is %s' %
+            (last_index + 1, location, filename))
+      valid = False
+    last_index = seqnum
+  return valid
+
+
+def verify_flyway_index():
+  """Verifies that the Flyway index file is in sync with the directory."""
+  success = True
+
+  # Sort the files in the Flyway directory by their sequence number.
+  files = sorted(
+      files_by_seqnum(os.listdir('db/src/main/resources/sql/flyway'),
+                      'Flyway directory'))
+
+  # Make sure that there are no gaps and no duplicate sequence numbers in the
+  # files themselves.
+  if not has_valid_order(files, 'Flyway directory'):
+    success = False
+
+  # Remove the sequence numbers and compare against the index file contents.
+  files = [filename[1] for filename in sorted(files)]
+  with open('db/src/main/resources/sql/flyway.txt') as index:
+    indexed_files = index.read().splitlines()
+  if files != indexed_files:
+    unindexed = set(files) - set(indexed_files)
+    if unindexed:
+      print('The following Flyway files are not in flyway.txt: %s' % unindexed)
+
+    nonexistent = set(indexed_files) - set(files)
+    if nonexistent:
+      print('The following files are in flyway.txt but not in the Flyway '
+            'directory: %s' % nonexistent)
+
+    # Do an ordering check on the index file (ignore the result, we're failing
+    # anyway).
+    has_valid_order(files_by_seqnum(indexed_files, 'flyway.txt'), 'flyway.txt')
+    success = False
+
+  if not success:
+    print('Please fix any conflicts and run "./nom_build :db:generateFlywayIndex"')
+
+  return not success
+
 
 def get_files():
   for root, dirnames, filenames in os.walk("."):
@@ -197,5 +282,10 @@ if __name__ == "__main__":
       failed = True
       print("%s had errors: \n  %s" % (file, "\n  ".join(error_messages)))
 
+  # And now for something completely different: check to see if the Flyway
+  # index is up-to-date.  It's quicker to do it here than in the unit tests:
+  # when we put it here it fails fast before all of the tests are run.
+  failed |= verify_flyway_index()
+
   if failed:
     sys.exit(1)

core/build.gradle

@@ -63,8 +63,6 @@ def dockerIncompatibleTestPatterns = [
     // methods, so we exclude the whole test class.
     "google/registry/tools/params/PathParameterTest.*",
     "google/registry/persistence/PersistenceModuleTest.*",
-    // This test is failing in docker when using Java 11. The cause is unclear.
-    "google/registry/tools/DomainLockUtilsTest.*",
 ]
 
 // Tests that conflict with members of both the main test suite and the
@@ -223,6 +221,7 @@ dependencies {
   compile deps['com.jcraft:jsch']
   testCompile deps['com.thoughtworks.qdox:qdox']
   compile deps['dnsjava:dnsjava']
+  runtime deps['guru.nidi:graphviz-java-all-j2v8']
   testCompile deps['io.github.classgraph:classgraph']
   testRuntime deps['io.github.java-diff-utils:java-diff-utils']
   testCompile deps['javax.annotation:javax.annotation-api']
@@ -247,6 +246,7 @@ dependencies {
   testCompile deps['org.apache.ftpserver:ftpserver-core']
   compile deps['org.apache.httpcomponents:httpclient']
   compile deps['org.apache.httpcomponents:httpcore']
+  runtime deps['org.apache.logging.log4j:log4j-core']
   testCompile deps['org.apache.sshd:sshd-core']
   testCompile deps['org.apache.sshd:sshd-scp']
   testCompile deps['org.apache.sshd:sshd-sftp']
@@ -259,6 +259,7 @@ dependencies {
   compile deps['org.hibernate:hibernate-core']
   compile deps['org.joda:joda-money']
   compile deps['org.json:json']
+  compile deps['org.jsoup:jsoup']
   testCompile deps['org.mortbay.jetty:jetty']
   compile deps['org.postgresql:postgresql']
   testCompile deps['org.seleniumhq.selenium:selenium-api']
@@ -271,6 +272,10 @@ dependencies {
   compile deps['org.testcontainers:postgresql']
   testCompile deps['org.testcontainers:selenium']
   testCompile deps['org.testcontainers:testcontainers']
+  compile deps['us.fatehi:schemacrawler']
+  compile deps['us.fatehi:schemacrawler-api']
+  compile deps['us.fatehi:schemacrawler-diagram']
+  compile deps['us.fatehi:schemacrawler-tools']
   compile deps['xerces:xmlParserAPIs']
   compile deps['xpp3:xpp3']
   // This dependency must come after javax.mail:mail as it would otherwise
@@ -635,131 +640,6 @@ artifacts {
   testRuntime testJar
 }
 
-/**
- * We have to break out the test suites because some of the tests conflict
- * with one another, but unfortunately this breaks the "--tests" flag.  The
- * --tests flag only applies to the task named on the command line (usually
- * just "test"), not for all tasks of type "Test".
- *
- * As a better solution, FilteringTest sets testNameIncludePatterns (the
- * internal property that --tests sets) from the value of the "testFilter"
- * property, allowing us to filter across all the tests in core without
- * explicitly specifying a test task or causing errors because there are no
- * matching tests in the main task.
- *
- * To use it, define "testFilter" to be a comma-separated collection of class
- * names (wildcards are allowed):
- *
- *   ./gradlew test -P testFilter=*.FooBar,google.registry.tools.ShellCommandTest
- */
-class FilteringTest extends Test {
-
-  FilteringTest() {
-    useJUnitPlatform();
-  }
-
-  private void applyTestFilter() {
-    if (project.testFilter) {
-      testNameIncludePatterns = project.testFilter.split(',')
-
-      // By default, gradle test tasks will produce a failure if no tests
-      // match the include/exclude/filter rules.  Since test filtering allows us
-      // to select a set of tests from a particular task, we don't want this
-      // behavior.
-      filter.failOnNoMatchingTests = false
-    }
-  }
-
-  /**
-   * Set to false if you also want to include TestCase and TestSuite classes.
-   *
-   * <p>Must be defined before "test", if at all.
-   */
-  boolean excludeTestCases = true
-
-  void setTests(List<String> tests) {
-    // Common exclude pattern. See README in parent directory for explanation.
-    if (excludeTestCases) {
-      exclude "**/*TestCase.*", "**/*TestSuite.*"
-    }
-    include tests
-    applyTestFilter()
-  }
-
-  /**
-   * Include all of the tests (except Test{Case,TestSuite}).  This actually
-   * doesn't explicitly "include" anything, in which cast the Test class tries
-   * to include everything that is not explicitly excluded.
-   */
-  void includeAllTests() {
-    exclude "**/*TestCase.*", "**/*TestSuite.*"
-    applyTestFilter()
-  }
-}
-
-task fragileTest(type: FilteringTest) {
-  // Common exclude pattern. See README in parent directory for explanation.
-  tests = fragileTestPatterns
-
-  if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") {
-    exclude dockerIncompatibleTestPatterns
-  }
-
-  // Run every test class in a freshly started process.
-  forkEvery 1
-
-  doFirst {
-    new File(screenshotsDir).deleteDir()
-  }
-}
-
-task outcastTest(type: FilteringTest) {
-  tests = outcastTestPatterns
-
-  // Sets the maximum number of test executors that may exist at the same time.
-  // Note that this number appears to contribute to NoClassDefFoundError
-  // exceptions on certain machines and distros. The root cause is unclear.
-  // Try reducing this number if you experience similar problems.
-  maxParallelForks 3
-}
-
-// Whitebox test verifying that RegistryTool can be instantiated. Note the
-// use of runtimeClasspath. This test emulates the logic in RegistryCli#run.
-// A to-do is added there to refactor.
-// TODO(weiminyu): Need a similar test for Registry server.
-task registryToolIntegrationTest {
-  dependsOn compileJava
-  doLast {
-    def classLoader =
-        new URLClassLoader(sourceSets.main.runtimeClasspath.collect {
-            it.toURI().toURL()
-        } as URL[])
-    def commandClasses =
-        (classLoader.loadClass('google.registry.tools.RegistryTool')
-            .getDeclaredField('COMMAND_MAP').get(null) as Map).values()
-
-    commandClasses.each {
-      try {
-        Constructor<?> c = ((Class<?>) it).getDeclaredConstructor()
-        c.setAccessible(true)
-        c.newInstance()
-      } catch (Throwable e) {
-        throw new RuntimeException("Failed to instantiate ${it}:\n ${e}")
-      }
-    }
-  }
-}
-
-// Dedicated test suite for schema-dependent tests.
-task sqlIntegrationTest(type: FilteringTest) {
-  // TestSuite still requires a JUnit 4 runner, which knows how to handle JUnit 5 tests.
-  // Here we need to override parent's choice of JUnit 5. If changing this, remember to
-  // change :integration:sqlIntegrationTest too.
-  useJUnit()
-  excludeTestCases = false
-  tests = ['google/registry/schema/integration/SqlIntegrationTestSuite.*']
-}
-
 task findGoldenImages(type: JavaExec) {
   classpath = sourceSets.test.runtimeClasspath
   main = 'google.registry.webdriver.GoldenImageFinder'
@@ -879,39 +759,6 @@ task generateGoldenImages(type: FilteringTest) {
 }
 generateGoldenImages.finalizedBy(findGoldenImages)
 
-task standardTest(type: FilteringTest) {
-  includeAllTests()
-  exclude fragileTestPatterns
-  exclude outcastTestPatterns
-  // See SqlIntegrationTestSuite.java
-  exclude '**/*BeforeSuiteTest.*', '**/*AfterSuiteTest.*'
-
-  if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") {
-    exclude dockerIncompatibleTestPatterns
-  }
-
-  // Run every test class in its own process.
-  // Uncomment to unblock build while troubleshooting inexplicable test errors.
-  // This setting makes the build take 35 minutes, without it it takes about 10.
-  // forkEvery 1
-
-  // Sets the maximum number of test executors that may exist at the same time.
-  // Also, Gradle executes tests in 1 thread and some of our test infrastructures
-  // depend on that, e.g. DualDatabaseTestInvocationContextProvider injects
-  // different implementation of TransactionManager into TransactionManagerFactory.
-  maxParallelForks 5
-
-  systemProperty 'test.projectRoot', rootProject.projectRootDir
-  systemProperty 'test.resourcesDir', resourcesDir
-}
-
-test {
-  // Don't run any tests from this task, all testing gets done in the
-  // FilteringTest tasks.
-  exclude "**"
-  // TODO(weiminyu): Remove dependency on sqlIntegrationTest
-}.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
-
 createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
 
 // A jar with classes and resources from main sourceSet, excluding internal
@@ -1046,6 +893,147 @@ task runTestServer(dependsOn: copyJsFilesForTestServer, type: JavaExec) {
   classpath = sourceSets.test.runtimeClasspath
 }
 
+/**
+ * We have to break out the test suites because some of the tests conflict
+ * with one another, but unfortunately this breaks the "--tests" flag.  The
+ * --tests flag only applies to the task named on the command line (usually
+ * just "test"), not for all tasks of type "Test".
+ *
+ * As a better solution, FilteringTest sets testNameIncludePatterns (the
+ * internal property that --tests sets) from the value of the "testFilter"
+ * property, allowing us to filter across all the tests in core without
+ * explicitly specifying a test task or causing errors because there are no
+ * matching tests in the main task.
+ *
+ * To use it, define "testFilter" to be a comma-separated collection of class
+ * names (wildcards are allowed):
+ *
+ *   ./gradlew test -P testFilter=*.FooBar,google.registry.tools.ShellCommandTest
+ */
+class FilteringTest extends Test {
+
+  FilteringTest() {
+    useJUnitPlatform();
+  }
+
+  private void applyTestFilter() {
+    if (project.testFilter) {
+      testNameIncludePatterns = project.testFilter.split(',')
+
+      // By default, gradle test tasks will produce a failure if no tests
+      // match the include/exclude/filter rules.  Since test filtering allows us
+      // to select a set of tests from a particular task, we don't want this
+      // behavior.
+      filter.failOnNoMatchingTests = false
+    }
+  }
+
+  /**
+   * Set to false if you also want to include TestCase and TestSuite classes.
+   *
+   * <p>Must be defined before "test", if at all.
+   */
+  boolean excludeTestCases = true
+
+  void setTests(List<String> tests) {
+    // Common exclude pattern. See README in parent directory for explanation.
+    if (excludeTestCases) {
+      exclude "**/*TestCase.*", "**/*TestSuite.*"
+    }
+    include tests
+    applyTestFilter()
+  }
+
+  /**
+   * Include all of the tests (except Test{Case,TestSuite}).  This actually
+   * doesn't explicitly "include" anything, in which cast the Test class tries
+   * to include everything that is not explicitly excluded.
+   */
+  void includeAllTests() {
+    exclude "**/*TestCase.*", "**/*TestSuite.*"
+    applyTestFilter()
+  }
+}
+
+task fragileTest(type: FilteringTest) {
+  // Common exclude pattern. See README in parent directory for explanation.
+  tests = fragileTestPatterns
+
+  if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") {
+    exclude dockerIncompatibleTestPatterns
+  }
+
+  // Run every test class in a freshly started process.
+  forkEvery 1
+
+  doFirst {
+    new File(screenshotsDir).deleteDir()
+  }
+}
+
+task outcastTest(type: FilteringTest) {
+  tests = outcastTestPatterns
+
+  // Sets the maximum number of test executors that may exist at the same time.
+  // Note that this number appears to contribute to NoClassDefFoundError
+  // exceptions on certain machines and distros. The root cause is unclear.
+  // Try reducing this number if you experience similar problems.
+  maxParallelForks 3
+}
+
+// Dedicated test suite for schema-dependent tests.
+task sqlIntegrationTest(type: FilteringTest) {
+  // TestSuite still requires a JUnit 4 runner, which knows how to handle JUnit 5 tests.
+  // Here we need to override parent's choice of JUnit 5. If changing this, remember to
+  // change :integration:sqlIntegrationTest too.
+  useJUnit()
+  excludeTestCases = false
+  tests = ['google/registry/schema/integration/SqlIntegrationTestSuite.*']
+}
+
+// Verifies that RegistryTool can be instantiated:
+// - All dependencies are packaged in nomulus.jar
+// - JPA setup succeeds.
+task registryToolIntegrationTest(dependsOn: nomulus, type: FilteringTest) {
+  tests = ['google/registry/tools/RegistryToolTest.*']
+  testClassesDirs = sourceSets.test.output.classesDirs
+  classpath = nomulus.outputs.files.plus(configurations.testRuntimeClasspath)
+      .plus(files(testClassesDirs))
+}
+
+task standardTest(type: FilteringTest) {
+  includeAllTests()
+  exclude fragileTestPatterns
+  exclude outcastTestPatterns
+  // See SqlIntegrationTestSuite.java
+  exclude '**/*BeforeSuiteTest.*', '**/*AfterSuiteTest.*'
+
+  if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") {
+    exclude dockerIncompatibleTestPatterns
+  }
+
+  // Run every test class in its own process.
+  // Uncomment to unblock build while troubleshooting inexplicable test errors.
+  // This setting makes the build take 35 minutes, without it it takes about 10.
+  // forkEvery 1
+
+  // Sets the maximum number of test executors that may exist at the same time.
+  // Also, Gradle executes tests in 1 thread and some of our test infrastructures
+  // depend on that, e.g. DualDatabaseTestInvocationContextProvider injects
+  // different implementation of TransactionManager into TransactionManagerFactory.
+  maxParallelForks 5
+
+  systemProperty 'test.projectRoot', rootProject.projectRootDir
+  systemProperty 'test.resourcesDir', resourcesDir
+}
+
+test {
+  // Don't run any tests from this task, all testing gets done in the
+  // FilteringTest tasks.
+  exclude "**"
+  // TODO(weiminyu): Remove dependency on sqlIntegrationTest
+}.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
+
 project.build.dependsOn devtool
 project.build.dependsOn buildToolImage
 project.build.dependsOn ':stage'

core/gradle/dependency-locks/annotationProcessor.lockfile

@@ -1,8 +1,8 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto.value:auto-value:1.6.3
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
@@ -11,11 +11,11 @@ com.google.dagger:dagger-compiler:2.28
 com.google.dagger:dagger-producers:2.28
 com.google.dagger:dagger-spi:2.28
 com.google.dagger:dagger:2.28
-com.google.errorprone:error_prone_annotation:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
 com.google.errorprone:error_prone_annotations:2.3.4
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
@@ -30,11 +30,14 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:2.11.1
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.3.61
 org.jetbrains.kotlin:kotlin-stdlib:1.3.61
 org.jetbrains.kotlinx:kotlinx-metadata-jvm:0.1.0
 org.jetbrains:annotations:13.0
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

core/gradle/dependency-locks/compile.lockfile

@@ -202,6 +202,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +226,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -249,5 +251,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -200,6 +200,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -222,6 +223,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -246,5 +248,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/default.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -177,6 +184,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -198,11 +206,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -226,6 +237,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -240,6 +252,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -249,7 +263,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +235,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -238,6 +250,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/errorprone.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

core/gradle/dependency-locks/nonprodAnnotationProcessor.lockfile

@@ -1,24 +1,27 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotation:2.3.3
-com.google.errorprone:error_prone_annotations:2.3.3
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
+com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:27.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.1
 com.google.protobuf:protobuf-java:3.4.0
 com.googlecode.java-diff-utils:diffutils:1.3.0
-org.checkerframework:checker-qual:2.5.3
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -202,6 +202,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +226,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -249,5 +251,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -201,6 +201,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +225,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -248,5 +250,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/runtime.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +235,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -238,6 +250,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -1,8 +1,8 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.github.ben-manes.caffeine:caffeine:2.7.0
 com.github.kevinstern:software-and-algorithms:1.0
-com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google.auto.value:auto-value:1.6.3
 com.google.auto:auto-common:0.10
 com.google.code.findbugs:jFormatString:3.0.0
@@ -11,11 +11,11 @@ com.google.dagger:dagger-compiler:2.28
 com.google.dagger:dagger-producers:2.28
 com.google.dagger:dagger-spi:2.28
 com.google.dagger:dagger:2.28
-com.google.errorprone:error_prone_annotation:2.3.3
+com.google.errorprone:error_prone_annotation:2.3.4
 com.google.errorprone:error_prone_annotations:2.3.4
-com.google.errorprone:error_prone_check_api:2.3.3
-com.google.errorprone:error_prone_core:2.3.3
-com.google.errorprone:error_prone_type_annotations:2.3.3
+com.google.errorprone:error_prone_check_api:2.3.4
+com.google.errorprone:error_prone_core:2.3.4
+com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
@@ -30,11 +30,14 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:2.11.1
-org.checkerframework:dataflow:2.5.3
-org.checkerframework:javacutil:2.5.3
+org.checkerframework:checker-qual:3.0.0
+org.checkerframework:dataflow:3.0.0
+org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.3.61
 org.jetbrains.kotlin:kotlin-stdlib:1.3.61
 org.jetbrains.kotlinx:kotlinx-metadata-jvm:0.1.0
 org.jetbrains:annotations:13.0
 org.pcollections:pcollections:2.1.2
+org.plumelib:plume-util:1.0.6
+org.plumelib:reflection-util:0.0.2
+org.plumelib:require-javadoc:0.1.0

core/gradle/dependency-locks/testCompile.lockfile

@@ -6,9 +6,9 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.10.2
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.fasterxml.jackson.core:jackson-databind:2.10.2
+com.fasterxml.jackson.core:jackson-annotations:2.11.2
+com.fasterxml.jackson.core:jackson-core:2.11.2
+com.fasterxml.jackson.core:jackson-databind:2.11.2
 com.fasterxml:classmate:1.5.1
 com.github.jnr:jffi:1.2.23
 com.github.jnr:jnr-a64asm:1.0.0
@@ -248,6 +248,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -298,5 +299,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -6,9 +6,9 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.10.2
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.fasterxml.jackson.core:jackson-databind:2.10.2
+com.fasterxml.jackson.core:jackson-annotations:2.11.2
+com.fasterxml.jackson.core:jackson-core:2.11.2
+com.fasterxml.jackson.core:jackson-databind:2.11.2
 com.fasterxml:classmate:1.5.1
 com.github.jnr:jffi:1.2.23
 com.github.jnr:jnr-a64asm:1.0.0
@@ -247,6 +247,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -297,5 +298,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testRuntime.lockfile

@@ -6,9 +6,13 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.10.2
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.fasterxml.jackson.core:jackson-databind:2.10.2
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
+com.fasterxml.jackson.core:jackson-annotations:2.11.2
+com.fasterxml.jackson.core:jackson-core:2.11.2
+com.fasterxml.jackson.core:jackson-databind:2.11.2
 com.fasterxml:classmate:1.5.1
 com.github.jnr:jffi:1.2.23
 com.github.jnr:jnr-a64asm:1.0.0
@@ -138,6 +142,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.github.java-diff-utils:java-diff-utils:4.0
@@ -186,6 +193,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.13
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
@@ -216,7 +224,8 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -251,6 +260,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -289,6 +299,8 @@ org.seleniumhq.selenium:selenium-opera-driver:3.141.59
 org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -299,7 +311,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -6,9 +6,13 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.10.2
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.fasterxml.jackson.core:jackson-databind:2.10.2
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
+com.fasterxml.jackson.core:jackson-annotations:2.11.2
+com.fasterxml.jackson.core:jackson-core:2.11.2
+com.fasterxml.jackson.core:jackson-databind:2.11.2
 com.fasterxml:classmate:1.5.1
 com.github.jnr:jffi:1.2.23
 com.github.jnr:jnr-a64asm:1.0.0
@@ -138,6 +142,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.github.java-diff-utils:java-diff-utils:4.0
@@ -186,6 +193,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.13
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
@@ -216,7 +224,8 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -251,6 +260,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -289,6 +299,8 @@ org.seleniumhq.selenium:selenium-opera-driver:3.141.59
 org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -300,7 +312,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -169,16 +169,20 @@ public final class AsyncTaskEnqueuer {
         lock.getRelockDuration().isPresent(),
         "Lock with ID %s not configured for relock",
         lock.getRevisionId());
+    enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
+  }
+
+  /** Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked. */
+  void enqueueDomainRelock(Duration countdown, long lockRevisionId, int previousAttempts) {
     String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
     addTaskToQueueWithRetry(
         asyncActionsPushQueue,
         TaskOptions.Builder.withUrl(RelockDomainAction.PATH)
             .method(Method.POST)
             .header("Host", backendHostname)
-            .param(
-                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
-                String.valueOf(lock.getRevisionId()))
-            .countdownMillis(lock.getRelockDuration().get().getMillis()));
+            .param(RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM, String.valueOf(lockRevisionId))
+            .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, String.valueOf(previousAttempts))
+            .countdownMillis(countdown.getMillis()));
   }
 
   /**

core/src/main/java/google/registry/batch/BatchModule.java

@@ -21,6 +21,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
+import static google.registry.request.RequestParameters.extractIntParameter;
 import static google.registry.request.RequestParameters.extractLongParameter;
 import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
 import static google.registry.request.RequestParameters.extractOptionalIntParameter;
@@ -94,9 +95,15 @@ public class BatchModule {
   }
 
   @Provides
-  @Parameter("oldUnlockRevisionId")
+  @Parameter(RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM)
   static long provideOldUnlockRevisionId(HttpServletRequest req) {
-    return extractLongParameter(req, "oldUnlockRevisionId");
+    return extractLongParameter(req, RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM);
+  }
+
+  @Provides
+  @Parameter(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM)
+  static int providePreviousAttempts(HttpServletRequest req) {
+    return extractIntParameter(req, RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM);
   }
 
   @Provides

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -464,7 +464,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
         HostResource host = (HostResource) existingResource;
         if (host.isSubordinate()) {
           dnsQueue.addHostRefreshTask(host.getHostName());
-          tm().saveNewOrUpdate(
+          tm().put(
                   tm().load(host.getSuperordinateDomain())
                       .asBuilder()
                       .removeSubordinateHost(host.getHostName())

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -15,19 +15,23 @@
 package google.registry.batch;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.StatusValue;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
 import google.registry.model.registry.RegistryLockDao;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
@@ -36,11 +40,15 @@ import google.registry.request.auth.Auth;
 import google.registry.schema.domain.RegistryLock;
 import google.registry.tools.DomainLockUtils;
 import google.registry.util.DateTimeUtils;
+import google.registry.util.EmailMessage;
+import google.registry.util.SendEmailService;
+import java.util.Optional;
 import javax.inject.Inject;
+import javax.mail.internet.AddressException;
+import javax.mail.internet.InternetAddress;
+import org.joda.time.Duration;
 
-/**
- * Task that relocks a previously-Registry-Locked domain after some predetermined period of time.
- */
+/** Task that re-locks a previously-Registry-Locked domain after a predetermined period of time. */
 @Action(
     service = Action.Service.BACKEND,
     path = RelockDomainAction.PATH,
@@ -51,30 +59,78 @@ public class RelockDomainAction implements Runnable {
 
   public static final String PATH = "/_dr/task/relockDomain";
   public static final String OLD_UNLOCK_REVISION_ID_PARAM = "oldUnlockRevisionId";
+  public static final String PREVIOUS_ATTEMPTS_PARAM = "previousAttempts";
+
+  static final int ATTEMPTS_BEFORE_SLOWDOWN = 36; // every ten minutes for six hours then every hour
+  static final int FAILURES_BEFORE_EMAIL = 2; // email after three failures, one half hour
+  private static final Duration TEN_MINUTES = Duration.standardMinutes(10);
+  private static final Duration ONE_HOUR = Duration.standardHours(1);
+
+  private static final String RELOCK_SUCCESS_EMAIL_TEMPLATE =
+      "The domain %s was successfully re-locked.\n\nPlease contact support at %s if you have any "
+          + "questions.";
+  private static final String RELOCK_NON_RETRYABLE_FAILURE_EMAIL_TEMPLATE =
+      "There was an error when automatically re-locking %s. Error message: %s\n\nPlease contact "
+          + "support at %s if you have any questions.";
+  private static final String RELOCK_TRANSIENT_FAILURE_EMAIL_TEMPLATE =
+      "There was an unexpected error when automatically re-locking %s. We will continue retrying "
+          + "the lock for five hours. Please contact support at %s if you have any questions";
+  private static final String RELOCK_UNKNOWN_ID_FAILURE_EMAIL_TEMPLATE =
+      "The old lock with revision ID %d is not present or is not accessible";
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   private final long oldUnlockRevisionId;
+  private final int previousAttempts;
+  private final InternetAddress alertRecipientAddress;
+  private final InternetAddress gSuiteOutgoingEmailAddress;
+  private final String supportEmail;
+  private final SendEmailService sendEmailService;
   private final DomainLockUtils domainLockUtils;
   private final Response response;
+  private final AsyncTaskEnqueuer asyncTaskEnqueuer;
 
   @Inject
   public RelockDomainAction(
       @Parameter(OLD_UNLOCK_REVISION_ID_PARAM) long oldUnlockRevisionId,
+      @Parameter(PREVIOUS_ATTEMPTS_PARAM) int previousAttempts,
+      @Config("alertRecipientEmailAddress") InternetAddress alertRecipientAddress,
+      @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress,
+      @Config("supportEmail") String supportEmail,
+      SendEmailService sendEmailService,
       DomainLockUtils domainLockUtils,
-      Response response) {
+      Response response,
+      AsyncTaskEnqueuer asyncTaskEnqueuer) {
     this.oldUnlockRevisionId = oldUnlockRevisionId;
+    this.previousAttempts = previousAttempts;
+    this.alertRecipientAddress = alertRecipientAddress;
+    this.gSuiteOutgoingEmailAddress = gSuiteOutgoingEmailAddress;
+    this.supportEmail = supportEmail;
+    this.sendEmailService = sendEmailService;
     this.domainLockUtils = domainLockUtils;
     this.response = response;
+    this.asyncTaskEnqueuer = asyncTaskEnqueuer;
   }
 
   @Override
   public void run() {
-    jpaTm().transact(this::relockDomain);
+    /* We wish to manually control our retry behavior, in order to limit the number of retries
+     * and/or notify registrars / support only after a certain number of retries, or only
+     * with a certain type of failure. AppEngine will automatically retry on any non-2xx status
+     * code, so return SC_NO_CONTENT (204) by default to avoid this auto-retry.
+     *
+     * See https://cloud.google.com/appengine/docs/standard/java/taskqueue/push/retrying-tasks
+     * for more details on retry behavior. */
+    response.setStatus(SC_NO_CONTENT);
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+
+    // nb: DomainLockUtils relies on the JPA transaction being the outermost transaction
+    jpaTm().transact(() -> tm().transact(this::relockDomain));
   }
 
   private void relockDomain() {
-    RegistryLock oldLock;
+    RegistryLock oldLock = null;
+    DomainBase domain;
     try {
       oldLock =
           RegistryLockDao.getByRevisionId(oldUnlockRevisionId)
@@ -82,87 +138,187 @@ public class RelockDomainAction implements Runnable {
                   () ->
                       new IllegalArgumentException(
                           String.format("Unknown revision ID %d", oldUnlockRevisionId)));
-      DomainBase domain =
+      domain =
           ofy()
               .load()
               .type(DomainBase.class)
               .id(oldLock.getRepoId())
               .now()
               .cloneProjectedAtTime(jpaTm().getTransactionTime());
-
-      if (domain.getStatusValues().containsAll(REGISTRY_LOCK_STATUSES)
-          || oldLock.getRelock() != null) {
-        // The domain was manually locked, so we shouldn't worry about relocking
-        String message =
-            String.format(
-                "Domain %s is already manually relocked, skipping automated relock.",
-                domain.getDomainName());
-        logger.atInfo().log(message);
-        // SC_NO_CONTENT (204) skips retry -- see the comment below
-        response.setStatus(SC_NO_CONTENT);
-        response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
-        response.setPayload(message);
-        return;
-      }
-      verifyDomainAndLockState(oldLock, domain);
     } catch (Throwable t) {
-      /* If there's a bad verification code or the domain is in a bad state, we won't want to retry.
-       * AppEngine will retry on non-2xx error codes, so we return SC_NO_CONTENT (204) to avoid it.
-       *
-       * See https://cloud.google.com/appengine/docs/standard/java/taskqueue/push/retrying-tasks
-       * for more details on retry behavior. */
-      logger.atWarning().withCause(t).log(
-          "Exception when attempting to relock domain with old revision ID %d.",
-          oldUnlockRevisionId);
-      response.setStatus(SC_NO_CONTENT);
-      response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
-      response.setPayload(String.format("Relock failed: %s", t.getMessage()));
+      handleTransientFailure(Optional.ofNullable(oldLock), t);
       return;
     }
-    applyRelock(oldLock);
+
+    if (domain.getStatusValues().containsAll(REGISTRY_LOCK_STATUSES)
+        || oldLock.getRelock() != null) {
+      // The domain was manually locked, so we shouldn't worry about re-locking
+      String message =
+          String.format(
+              "Domain %s is already manually re-locked, skipping automated re-lock.",
+              domain.getDomainName());
+      logger.atInfo().log(message);
+      response.setPayload(message);
+      return;
+    }
+    try {
+      verifyDomainAndLockState(oldLock, domain);
+    } catch (Throwable t) {
+      // If the domain was, for example, transferred, then notify the old registrar and don't retry.
+      handleNonRetryableFailure(oldLock, t);
+      return;
+    }
+    try {
+      applyRelock(oldLock);
+    } catch (Throwable t) {
+      handleTransientFailure(Optional.of(oldLock), t);
+    }
   }
 
   private void applyRelock(RegistryLock oldLock) {
-    try {
-      domainLockUtils.administrativelyApplyLock(
-          oldLock.getDomainName(),
-          oldLock.getRegistrarId(),
-          oldLock.getRegistrarPocId(),
-          oldLock.isSuperuser());
-      logger.atInfo().log("Relocked domain %s.", oldLock.getDomainName());
-      response.setStatus(SC_OK);
-    } catch (Throwable t) {
-      // Any errors that occur here are unexpected, so we should retry. Return a non-2xx
-      // error code to get AppEngine to retry
-      logger.atSevere().withCause(t).log(
-          "Exception when attempting to relock domain %s.", oldLock.getDomainName());
-      response.setStatus(SC_INTERNAL_SERVER_ERROR);
-      response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
-      response.setPayload(String.format("Relock failed: %s", t.getMessage()));
+    domainLockUtils.administrativelyApplyLock(
+        oldLock.getDomainName(),
+        oldLock.getRegistrarId(),
+        oldLock.getRegistrarPocId(),
+        oldLock.isSuperuser());
+    logger.atInfo().log("Re-locked domain %s.", oldLock.getDomainName());
+    response.setStatus(SC_OK);
+    // Only send a success email if we previously sent a failure email
+    if (previousAttempts > FAILURES_BEFORE_EMAIL) {
+      sendSuccessEmail(oldLock);
     }
   }
 
   private void verifyDomainAndLockState(RegistryLock oldLock, DomainBase domain) {
     // Domain shouldn't be deleted or have a pending transfer/delete
     String domainName = domain.getDomainName();
-    checkArgument(
-        !DateTimeUtils.isAtOrAfter(jpaTm().getTransactionTime(), domain.getDeletionTime()),
-        "Domain %s has been deleted",
-        domainName);
     ImmutableSet<StatusValue> statusValues = domain.getStatusValues();
     checkArgument(
         !statusValues.contains(StatusValue.PENDING_DELETE),
-        "Domain %s has a pending delete",
+        "Domain %s has a pending delete.",
+        domainName);
+    checkArgument(
+        !DateTimeUtils.isAtOrAfter(jpaTm().getTransactionTime(), domain.getDeletionTime()),
+        "Domain %s has been deleted.",
         domainName);
     checkArgument(
         !statusValues.contains(StatusValue.PENDING_TRANSFER),
-        "Domain %s has a pending transfer",
+        "Domain %s has a pending transfer.",
         domainName);
     checkArgument(
         domain.getCurrentSponsorClientId().equals(oldLock.getRegistrarId()),
-        "Domain %s has been transferred from registrar %s to registrar %s since the unlock",
+        "Domain %s has been transferred from registrar %s to registrar %s since the unlock.",
         domainName,
         oldLock.getRegistrarId(),
         domain.getCurrentSponsorClientId());
   }
+
+  private void handleNonRetryableFailure(RegistryLock oldLock, Throwable t) {
+    logger.atWarning().withCause(t).log(
+        "Exception thrown when attempting to re-lock domain with old revision ID %d.",
+        oldUnlockRevisionId);
+    response.setPayload(String.format("Re-lock failed: %s", t.getMessage()));
+
+    String body =
+        String.format(
+            RELOCK_NON_RETRYABLE_FAILURE_EMAIL_TEMPLATE,
+            oldLock.getDomainName(),
+            t.getMessage(),
+            supportEmail);
+    sendEmailService.sendEmail(
+        EmailMessage.newBuilder()
+            .setFrom(gSuiteOutgoingEmailAddress)
+            .setBody(body)
+            .setSubject(String.format("Error re-locking domain %s", oldLock.getDomainName()))
+            .setRecipients(getEmailRecipients(oldLock.getRegistrarId()))
+            .build());
+  }
+
+  private void handleTransientFailure(Optional<RegistryLock> oldLock, Throwable t) {
+    String message = String.format("Re-lock failed: %s", t.getMessage());
+    logger.atSevere().withCause(t).log(message);
+    response.setPayload(message);
+
+    if (previousAttempts == FAILURES_BEFORE_EMAIL) {
+      if (oldLock.isPresent()) {
+        sendGenericTransientFailureEmail(oldLock.get());
+      } else {
+        // if the old lock isn't present, something has gone horribly wrong
+        sendUnknownRevisionIdAlertEmail();
+      }
+    }
+    Duration timeBeforeRetry = previousAttempts < ATTEMPTS_BEFORE_SLOWDOWN ? TEN_MINUTES : ONE_HOUR;
+    asyncTaskEnqueuer.enqueueDomainRelock(
+        timeBeforeRetry, oldUnlockRevisionId, previousAttempts + 1);
+  }
+
+  private void sendSuccessEmail(RegistryLock oldLock) {
+    String body =
+        String.format(RELOCK_SUCCESS_EMAIL_TEMPLATE, oldLock.getDomainName(), supportEmail);
+
+    sendEmailService.sendEmail(
+        EmailMessage.newBuilder()
+            .setFrom(gSuiteOutgoingEmailAddress)
+            .setBody(body)
+            .setSubject(String.format("Successful re-lock of domain %s", oldLock.getDomainName()))
+            .setRecipients(getEmailRecipients(oldLock.getRegistrarId()))
+            .build());
+  }
+
+  private void sendGenericTransientFailureEmail(RegistryLock oldLock) {
+    String body =
+        String.format(
+            RELOCK_TRANSIENT_FAILURE_EMAIL_TEMPLATE, oldLock.getDomainName(), supportEmail);
+    // For an unexpected failure, notify both the lock-enabled contacts and our alerting email
+    ImmutableSet<InternetAddress> allRecipients =
+        new ImmutableSet.Builder<InternetAddress>()
+            .addAll(getEmailRecipients(oldLock.getRegistrarId()))
+            .add(alertRecipientAddress)
+            .build();
+    sendEmailService.sendEmail(
+        EmailMessage.newBuilder()
+            .setFrom(gSuiteOutgoingEmailAddress)
+            .setBody(body)
+            .setSubject(String.format("Error re-locking domain %s", oldLock.getDomainName()))
+            .setRecipients(allRecipients)
+            .build());
+  }
+
+  private void sendUnknownRevisionIdAlertEmail() {
+    sendEmailService.sendEmail(
+        EmailMessage.newBuilder()
+            .setFrom(gSuiteOutgoingEmailAddress)
+            .setBody(String.format(RELOCK_UNKNOWN_ID_FAILURE_EMAIL_TEMPLATE, oldUnlockRevisionId))
+            .setSubject("Error re-locking domain")
+            .setRecipients(ImmutableSet.of(alertRecipientAddress))
+            .build());
+  }
+
+  private ImmutableSet<InternetAddress> getEmailRecipients(String registrarId) {
+    Registrar registrar =
+        Registrar.loadByClientIdCached(registrarId)
+            .orElseThrow(
+                () ->
+                    new IllegalStateException(String.format("Unknown registrar %s", registrarId)));
+
+    ImmutableSet<String> registryLockEmailAddresses =
+        registrar.getContacts().stream()
+            .filter(RegistrarContact::isRegistryLockAllowed)
+            .map(RegistrarContact::getRegistryLockEmailAddress)
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .collect(toImmutableSet());
+
+    ImmutableSet.Builder<InternetAddress> builder = new ImmutableSet.Builder<>();
+    // can't use streams due to the 'throws' in the InternetAddress constructor
+    for (String registryLockEmailAddress : registryLockEmailAddresses) {
+      try {
+        builder.add(new InternetAddress(registryLockEmailAddress));
+      } catch (AddressException e) {
+        // This shouldn't stop any other emails going out, so swallow it
+        logger.atWarning().log("Invalid email address %s", registryLockEmailAddress);
+      }
+    }
+    return builder.build();
+  }
 }

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -437,7 +437,7 @@ public final class Transforms {
                 .map(Optional::get)
                 .map(ofy::toPojo)
                 .collect(ImmutableList.toImmutableList());
-        retry(() -> jpaTm().transact(() -> jpaTm().saveNewOrUpdateAll(ofyEntities)));
+        retry(() -> jpaTm().transact(() -> jpaTm().putAll(ofyEntities)));
       }
     }
 

core/src/main/java/google/registry/beam/invoicing/sql/billing_events.sql

@@ -55,9 +55,7 @@ FROM (
     FROM
       `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
     WHERE
-    -- TODO(b/18092292): Add a filter for tldState (not PDT/PREDELEGATION)
-      tldType = 'REAL'
-    AND disableInvoicing is not TRUE) ) AS BillingEvent
+      invoicingEnabled IS TRUE) ) AS BillingEvent
   -- Gather billing ID from registrar table
   -- This is a 'JOIN' as opposed to 'LEFT JOIN' to filter out
   -- non-billable registrars

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -215,7 +215,7 @@ public class Spec11Pipeline implements Serializable {
                           .setRegistrarId(subdomain.registrarId())
                           .build();
                   JpaTransactionManager jpaTransactionManager = jpaSupplierFactory.get();
-                  jpaTransactionManager.transact(() -> jpaTransactionManager.saveNew(threatMatch));
+                  jpaTransactionManager.transact(() -> jpaTransactionManager.insert(threatMatch));
                 }
               }
             }));

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -16,9 +16,12 @@ package google.registry.config;
 
 import static com.google.common.base.Suppliers.memoize;
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap;
 import static google.registry.config.ConfigUtils.makeUrl;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.ResourceUtils.readResourceUtf8;
 import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import static java.util.Comparator.naturalOrder;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Ascii;
@@ -27,6 +30,7 @@ import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.util.TaskQueueUtils;
@@ -46,6 +50,7 @@ import javax.inject.Qualifier;
 import javax.inject.Singleton;
 import javax.mail.internet.AddressException;
 import javax.mail.internet.InternetAddress;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeConstants;
 import org.joda.time.Duration;
 
@@ -1345,6 +1350,33 @@ public final class RegistryConfig {
     public static String provideRdapTosStaticUrl(RegistryConfigSettings config) {
       return config.registryPolicy.rdapTosStaticUrl;
     }
+
+    @Provides
+    @Config("maxValidityDaysSchedule")
+    public static ImmutableSortedMap<DateTime, Integer> provideValidityDaysMap(
+        RegistryConfigSettings config) {
+      return config.sslCertificateValidation.maxValidityDaysSchedule.entrySet().stream()
+          .collect(
+              toImmutableSortedMap(
+                  naturalOrder(),
+                  e ->
+                      e.getKey().equals("START_OF_TIME")
+                          ? START_OF_TIME
+                          : DateTime.parse(e.getKey()),
+                  e -> e.getValue()));
+    }
+
+    @Provides
+    @Config("expirationWarningDays")
+    public static int provideDaysToExpiration(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.expirationWarningDays;
+    }
+
+    @Provides
+    @Config("minimumRsaKeyLength")
+    public static int provideMinimumRsaKeyLength(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.minimumRsaKeyLength;
+    }
   }
 
   /** Returns the App Engine project ID, which is based off the environment name. */

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -15,6 +15,7 @@
 package google.registry.config;
 
 import java.util.List;
+import java.util.Map;
 
 /** The POJO that YAML config files are deserialized into. */
 public class RegistryConfigSettings {
@@ -38,6 +39,7 @@ public class RegistryConfigSettings {
   public Beam beam;
   public Keyring keyring;
   public RegistryTool registryTool;
+  public SslCertificateValidation sslCertificateValidation;
 
   /** Configuration options that apply to the entire App Engine project. */
   public static class AppEngine {
@@ -218,4 +220,11 @@ public class RegistryConfigSettings {
     public String clientSecret;
     public String username;
   }
+
+  /** Configuration for the certificate checker. */
+  public static class SslCertificateValidation {
+    public Map<String, Integer> maxValidityDaysSchedule;
+    public int expirationWarningDays;
+    public int minimumRsaKeyLength;
+  }
 }

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -446,3 +446,17 @@ registryTool:
   # OAuth client secret used by the tool.
   clientSecret: YOUR_CLIENT_SECRET
   username: toolusername
+
+# Configuration options for checking SSL certificates.
+sslCertificateValidation:
+  # A map specifying the maximum amount of days the certificate can be valid.
+  # The entry key is the date closest before the date the certificate was issued
+  # and the entry value is the applicable maximum validity days for that certificate.
+  maxValidityDaysSchedule:
+    "START_OF_TIME": 825
+    "2020-09-01T00:00:00Z": 398
+  # The number of days before a certificate expires that indicates the
+  # certificate is nearing expiration and warnings should be sent.
+  expirationWarningDays: 30
+  # The minimum number of bits an RSA key must contain
+  minimumRsaKeyLength: 2048

core/src/main/java/google/registry/flows/certs/CertificateChecker.java

@@ -0,0 +1,221 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.flows.certs;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.Clock;
+import google.registry.util.DateTimeUtils;
+import java.io.ByteArrayInputStream;
+import java.security.PublicKey;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.security.interfaces.RSAPublicKey;
+import java.util.Date;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.Days;
+
+/** An utility to check that a given certificate meets our requirements */
+public class CertificateChecker {
+
+  private final ImmutableSortedMap<DateTime, Integer> maxValidityLengthSchedule;
+  private final int daysToExpiration;
+  private final int minimumRsaKeyLength;
+  private final Clock clock;
+
+  /**
+   * Constructs a CertificateChecker instance with the specified configuration parameters.
+   *
+   * <p>The max validity length schedule is a sorted map of {@link DateTime} to {@link Integer}
+   * entries representing a maximum validity period for certificates issued on or after that date.
+   * The first entry must have a key of {@link DateTimeUtils#START_OF_TIME}, such that every
+   * possible date has an applicable max validity period. Since security requirements tighten over
+   * time, the max validity periods will be decreasing as the date increases.
+   *
+   * <p>The validity length schedule used by all major Web browsers as of 2020Q4 would be
+   * represented as:
+   *
+   * <pre>
+   *   ImmutableSortedMap.of(
+   *     START_OF_TIME, 825,
+   *     DateTime.parse("2020-09-01T00:00:00Z"), 398
+   *   );
+   * </pre>
+   */
+  @Inject
+  public CertificateChecker(
+      @Config("maxValidityDaysSchedule")
+          ImmutableSortedMap<DateTime, Integer> maxValidityLengthSchedule,
+      @Config("expirationWarningDays") int daysToExpiration,
+      @Config("minimumRsaKeyLength") int minimumRsaKeyLength,
+      Clock clock) {
+    checkArgument(
+        maxValidityLengthSchedule.containsKey(START_OF_TIME),
+        "Max validity length schedule must contain an entry for START_OF_TIME");
+    this.maxValidityLengthSchedule = maxValidityLengthSchedule;
+    this.daysToExpiration = daysToExpiration;
+    this.minimumRsaKeyLength = minimumRsaKeyLength;
+    this.clock = clock;
+  }
+
+  /**
+   * Checks the given certificate string for violations and throws an exception if any violations
+   * exist.
+   */
+  public void validateCertificate(String certificateString) {
+    ImmutableSet<CertificateViolation> violations = checkCertificate(certificateString);
+    if (!violations.isEmpty()) {
+      String displayMessages =
+          violations.stream()
+              .map(violation -> getViolationDisplayMessage(violation))
+              .collect(Collectors.joining("\n"));
+      throw new IllegalArgumentException(displayMessages);
+    }
+  }
+
+  /**
+   * Checks a given certificate for violations and returns a list of all the violations the
+   * certificate has.
+   */
+  public ImmutableSet<CertificateViolation> checkCertificate(X509Certificate certificate) {
+    ImmutableSet.Builder<CertificateViolation> violations = new ImmutableSet.Builder<>();
+
+    // Check if currently in validity period
+    Date now = clock.nowUtc().toDate();
+    if (certificate.getNotAfter().before(now)) {
+      violations.add(CertificateViolation.EXPIRED);
+    } else if (certificate.getNotBefore().after(now)) {
+      violations.add(CertificateViolation.NOT_YET_VALID);
+    }
+
+    // Check validity period length
+    int maxValidityDays =
+        maxValidityLengthSchedule.floorEntry(new DateTime(certificate.getNotBefore())).getValue();
+    if (getValidityLengthInDays(certificate) > maxValidityDays) {
+      violations.add(CertificateViolation.VALIDITY_LENGTH_TOO_LONG);
+    }
+
+    // Check key strengths
+    PublicKey key = certificate.getPublicKey();
+    if (key.getAlgorithm().equals("RSA")) {
+      RSAPublicKey rsaPublicKey = (RSAPublicKey) key;
+      if (rsaPublicKey.getModulus().bitLength() < minimumRsaKeyLength) {
+        violations.add(CertificateViolation.RSA_KEY_LENGTH_TOO_SHORT);
+      }
+    } else if (key.getAlgorithm().equals("EC")) {
+      // TODO(sarahbot): Add verification of ECDSA curves
+    } else {
+      violations.add(CertificateViolation.ALGORITHM_CONSTRAINED);
+    }
+    return violations.build();
+  }
+
+  /**
+   * Converts a given string to a certificate and checks it for violations, returning a list of all
+   * the violations the certificate has.
+   */
+  public ImmutableSet<CertificateViolation> checkCertificate(String certificateString) {
+    X509Certificate certificate;
+
+    try {
+      certificate =
+          (X509Certificate)
+              CertificateFactory.getInstance("X509")
+                  .generateCertificate(new ByteArrayInputStream(certificateString.getBytes(UTF_8)));
+    } catch (CertificateException e) {
+      throw new IllegalArgumentException("Unable to read given certificate.");
+    }
+
+    return checkCertificate(certificate);
+  }
+
+  /**
+   * Returns whether the certificate is nearing expiration.
+   *
+   * <p>Note that this is <i>all</i> that it checks. The certificate itself may well be expired or
+   * not yet valid and this message will still return false. So you definitely want to pair a call
+   * to this method with a call to {@link #checkCertificate} to determine other issues with the
+   * certificate that may be occurring.
+   */
+  public boolean isNearingExpiration(X509Certificate certificate) {
+    Date nearingExpirationDate =
+        DateTime.parse(certificate.getNotAfter().toInstant().toString())
+            .minusDays(daysToExpiration)
+            .toDate();
+    return clock.nowUtc().toDate().after(nearingExpirationDate);
+  }
+
+  private static int getValidityLengthInDays(X509Certificate certificate) {
+    DateTime start = DateTime.parse(certificate.getNotBefore().toInstant().toString());
+    DateTime end = DateTime.parse(certificate.getNotAfter().toInstant().toString());
+    return Days.daysBetween(start.withTimeAtStartOfDay(), end.withTimeAtStartOfDay()).getDays();
+  }
+
+  private String getViolationDisplayMessage(CertificateViolation certificateViolation) {
+    // Yes, we'd rather do this as an instance method on the CertificateViolation enum itself, but
+    // we can't because we need access to configuration (injected as instance variables) which you
+    // can't get in a static enum context.
+    switch (certificateViolation) {
+      case EXPIRED:
+        return "Certificate is expired.";
+      case NOT_YET_VALID:
+        return "Certificate start date is in the future.";
+      case ALGORITHM_CONSTRAINED:
+        return "Certificate key algorithm must be RSA or ECDSA.";
+      case RSA_KEY_LENGTH_TOO_SHORT:
+        return String.format(
+            "RSA key length is too short; the minimum allowed length is %d bits.",
+            this.minimumRsaKeyLength);
+      case VALIDITY_LENGTH_TOO_LONG:
+        return String.format(
+            "Certificate validity period is too long; it must be less than or equal to %d days.",
+            this.maxValidityLengthSchedule.lastEntry().getValue());
+      default:
+        throw new IllegalArgumentException(
+            String.format(
+                "Unknown CertificateViolation enum value: %s", certificateViolation.name()));
+    }
+  }
+
+  /**
+   * The type of violation a certificate has based on the certificate requirements
+   * (go/registry-proxy-security).
+   */
+  public enum CertificateViolation {
+    EXPIRED,
+    NOT_YET_VALID,
+    VALIDITY_LENGTH_TOO_LONG,
+    RSA_KEY_LENGTH_TOO_SHORT,
+    ALGORITHM_CONSTRAINED;
+
+    /**
+     * Gets a suitable end-user-facing display message for this particular certificate violation.
+     *
+     * <p>Note that the {@link CertificateChecker} instance must be passed in because it contains
+     * configuration values (e.g. minimum RSA key length) that go into the error message text.
+     */
+    public String getDisplayMessage(CertificateChecker certificateChecker) {
+      return certificateChecker.getViolationDisplayMessage(this);
+    }
+  }
+}

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -370,7 +370,8 @@ public class DomainCreateFlow implements TransactionalFlow {
     if (allocationToken.isPresent()
         && TokenType.SINGLE_USE.equals(allocationToken.get().getTokenType())) {
       entitiesToSave.add(
-          allocationTokenFlowUtils.redeemToken(allocationToken.get(), Key.create(historyEntry)));
+          allocationTokenFlowUtils.redeemToken(
+              allocationToken.get(), HistoryEntry.createVKey(Key.create(historyEntry))));
     }
     enqueueTasks(newDomain, hasSignedMarks, hasClaimsNotice);
 

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -173,7 +173,7 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
             existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId);
     updateForeignKeyIndexDeletionTime(newDomain);
     entitiesToSave.add(newDomain, historyEntry, autorenewEvent, autorenewPollMessage);
-    tm().saveNewOrUpdateAll(entitiesToSave.build());
+    tm().putAll(entitiesToSave.build());
     tm().delete(existingDomain.getDeletePollMessage());
     dnsQueue.addDomainRefreshTask(existingDomain.getDomainName());
     return responseBuilder

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -67,7 +67,9 @@ import google.registry.model.domain.DomainCommand.Update.AddRemove;
 import google.registry.model.domain.DomainCommand.Update.Change;
 import google.registry.model.domain.fee.FeeUpdateCommandExtension;
 import google.registry.model.domain.metadata.MetadataExtension;
+import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.domain.secdns.SecDnsUpdateExtension;
+import google.registry.model.domain.superuser.DomainUpdateSuperuserExtension;
 import google.registry.model.eppcommon.AuthInfo;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
@@ -152,7 +154,8 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     extensionManager.register(
         FeeUpdateCommandExtension.class,
         MetadataExtension.class,
-        SecDnsUpdateExtension.class);
+        SecDnsUpdateExtension.class,
+        DomainUpdateSuperuserExtension.class);
     flowCustomLogic.beforeValidation();
     extensionManager.validate();
     validateClientIsLoggedIn(clientId);
@@ -236,10 +239,16 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     DomainBase.Builder domainBuilder =
         domain
             .asBuilder()
-            // Handle the secDNS extension.
+            // Handle the secDNS extension. As dsData in secDnsUpdate is read from EPP input and
+            // does not have domainRepoId set, we create a copy of the existing dsData without
+            // domainRepoId for comparison.
             .setDsData(
                 secDnsUpdate.isPresent()
-                    ? updateDsData(domain.getDsData(), secDnsUpdate.get())
+                    ? updateDsData(
+                        domain.getDsData().stream()
+                            .map(DelegationSignerData::cloneWithoutDomainRepoId)
+                            .collect(toImmutableSet()),
+                        secDnsUpdate.get())
                     : domain.getDsData())
             .setLastEppUpdateTime(now)
             .setLastEppUpdateClientId(clientId)
@@ -251,6 +260,15 @@ public final class DomainUpdateFlow implements TransactionalFlow {
             .removeContacts(remove.getContacts())
             .setRegistrant(firstNonNull(change.getRegistrant(), domain.getRegistrant()))
             .setAuthInfo(firstNonNull(change.getAuthInfo(), domain.getAuthInfo()));
+    Optional<DomainUpdateSuperuserExtension> superuserExt =
+        eppInput.getSingleExtension(DomainUpdateSuperuserExtension.class);
+    if (superuserExt.isPresent()) {
+      if (superuserExt.get().getAutorenews().isPresent()) {
+        boolean autorenews = superuserExt.get().getAutorenews().get();
+        domainBuilder.setAutorenewEndTime(
+            Optional.ofNullable(autorenews ? null : domain.getRegistrationExpirationTime()));
+      }
+    }
     return domainBuilder.build();
   }
 

core/src/main/java/google/registry/flows/domain/token/AllocationTokenFlowUtils.java

@@ -33,6 +33,7 @@ import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -107,7 +108,7 @@ public class AllocationTokenFlowUtils {
 
   /** Redeems a SINGLE_USE {@link AllocationToken}, returning the redeemed copy. */
   public AllocationToken redeemToken(
-      AllocationToken token, Key<HistoryEntry> redemptionHistoryEntry) {
+      AllocationToken token, VKey<HistoryEntry> redemptionHistoryEntry) {
     checkArgument(
         TokenType.SINGLE_USE.equals(token.getTokenType()),
         "Only SINGLE_USE tokens can be marked as redeemed");
@@ -124,7 +125,8 @@ public class AllocationTokenFlowUtils {
   private void validateToken(
       InternetDomainName domainName, AllocationToken token, String clientId, DateTime now)
       throws EppException {
-    if (!token.getAllowedClientIds().isEmpty() && !token.getAllowedClientIds().contains(clientId)) {
+    if (!token.getAllowedRegistrarIds().isEmpty()
+        && !token.getAllowedRegistrarIds().contains(clientId)) {
       throw new AllocationTokenNotValidForRegistrarException();
     }
     if (!token.getAllowedTlds().isEmpty()

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -285,7 +285,7 @@ public final class HostUpdateFlow implements TransactionalFlow {
         && newHost.isSubordinate()
         && Objects.equals(
             existingHost.getSuperordinateDomain(), newHost.getSuperordinateDomain())) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(existingHost.getSuperordinateDomain())
                   .asBuilder()
                   .removeSubordinateHost(existingHost.getHostName())
@@ -294,14 +294,14 @@ public final class HostUpdateFlow implements TransactionalFlow {
       return;
     }
     if (existingHost.isSubordinate()) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(existingHost.getSuperordinateDomain())
                   .asBuilder()
                   .removeSubordinateHost(existingHost.getHostName())
                   .build());
     }
     if (newHost.isSubordinate()) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(newHost.getSuperordinateDomain())
                   .asBuilder()
                   .addSubordinateHost(newHost.getHostName())

core/src/main/java/google/registry/flows/poll/PollFlowUtils.java

@@ -64,7 +64,7 @@ public final class PollFlowUtils {
       // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
       // autorenew poll message has no more events to deliver and should be deleted.
       if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
-        tm().saveNewOrUpdate(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
+        tm().put(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
         includeAckedMessageInCount = isBeforeOrAt(nextEventTime, tm().getTransactionTime());
       } else {
         tm().delete(autorenewPollMessage.createVKey());

core/src/main/java/google/registry/idn/Arabic-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Arab
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Armenian-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Armn
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Bengali-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Beng
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Chinese-IDN.txt

@@ -1,24 +1,24 @@
 # Registry: Charleston Road Registry Inc.
-# Script: Chinese
+# Script: zh-Hans
 # Version: 1.0
 # Effective Date: 04-12-2012
 #
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com
 # Notes: This table describes codepoints allowed for the Chinese script.
 
-Reference 0 Unicode 3.2
-Reference 1 A Complete Set of Simplified Chinese Characters
-Reference 2 Chinese Variants Collation Table
-Reference 3 Chinese Big Dictionary
-Reference 4 Chinese Relationship Table for Unihan Project
-Reference 5 GB2312
-Reference 6 General Table for Modern Chinese
-Reference 7 International Chinese Standard Big Dictionary
-Reference 8 Unihan Database
-Reference 9 BIG5
+# Reference 0 Unicode 3.2
+# Reference 1 A Complete Set of Simplified Chinese Characters
+# Reference 2 Chinese Variants Collation Table
+# Reference 3 Chinese Big Dictionary
+# Reference 4 Chinese Relationship Table for Unihan Project
+# Reference 5 GB2312
+# Reference 6 General Table for Modern Chinese
+# Reference 7 International Chinese Standard Big Dictionary
+# Reference 8 Unihan Database
+# Reference 9 BIG5
 
 U+002D(0);U+002D(0);
 U+0030(0);U+0030(0);

core/src/main/java/google/registry/idn/Cyrillic-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Cyrl
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Devanagari-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Deva
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Ethiopic-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Ethi
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Georgian-IDN.txt

@@ -3,7 +3,7 @@
 #
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Greek-IDN.txt

@@ -3,7 +3,7 @@
 # Script: Grek
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Gurmukhi-IDN.txt

@@ -3,7 +3,7 @@
 # Script: Guru
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Hebrew-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Hebr
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Kannada-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Knda
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Khmer-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Khmr
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 #
 # Telephone: +1 (650) 253-0000

core/src/main/java/google/registry/idn/Korean-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Kore
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Malayalam-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Mlym
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Myanmar-IDN.txt

@@ -3,7 +3,7 @@
 # Version: 1.0
 # Effective Date: 04-12-2012
 #
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Oriya-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Orya
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Sinhala-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Sinh
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Tamil-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Taml
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Telugu-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Telu
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Thai-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Thai
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Tibetan-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Tibt
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Traditional-Chinese-IDN.txt

@@ -1,23 +1,23 @@
 # Registry: Charleston Road Registry Inc.
-# Script: Traditional Chinese
+# Script: zh-Hant
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com
 # Notes: This table describes codepoints allowed for the Traditional Chinese script.
 
-Reference 0 Unicode 3.2
-Reference 1 A Complete Set of Simplified Chinese Characters
-Reference 2 Chinese Variants Collation Table
-Reference 3 Chinese Big Dictionary
-Reference 4 Chinese Relationship Table for Unihan Project
-Reference 5 GB2312
-Reference 6 General Table for Modern Chinese
-Reference 7 International Chinese Standard Big Dictionary
-Reference 8 Unihan Database
-Reference 9 BIG5
+# Reference 0 Unicode 3.2
+# Reference 1 A Complete Set of Simplified Chinese Characters
+# Reference 2 Chinese Variants Collation Table
+# Reference 3 Chinese Big Dictionary
+# Reference 4 Chinese Relationship Table for Unihan Project
+# Reference 5 GB2312
+# Reference 6 General Table for Modern Chinese
+# Reference 7 International Chinese Standard Big Dictionary
+# Reference 8 Unihan Database
+# Reference 9 BIG5
 
 U+002D(0);U+002D(0);
 U+0030(0);U+0030(0);

core/src/main/java/google/registry/model/EntityClasses.java

@@ -19,9 +19,12 @@ import google.registry.model.billing.BillingEvent;
 import google.registry.model.common.Cursor;
 import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.common.GaeUserIdConverter;
+import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.index.EppResourceIndexBucket;
@@ -68,9 +71,11 @@ public final class EntityClasses {
           CommitLogCheckpointRoot.class,
           CommitLogManifest.class,
           CommitLogMutation.class,
+          ContactHistory.class,
           ContactResource.class,
           Cursor.class,
           DomainBase.class,
+          DomainHistory.class,
           EntityGroupRoot.class,
           EppResourceIndex.class,
           EppResourceIndexBucket.class,
@@ -79,6 +84,7 @@ public final class EntityClasses {
           ForeignKeyIndex.ForeignKeyHostIndex.class,
           GaeUserIdConverter.class,
           HistoryEntry.class,
+          HostHistory.class,
           HostResource.class,
           KmsSecret.class,
           KmsSecretRevision.class,

core/src/main/java/google/registry/model/EppResource.java

@@ -24,7 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
@@ -62,23 +61,32 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   /**
    * Unique identifier in the registry for this resource.
    *
+   * <p>Not persisted so that we can store these in references to other objects. Subclasses that
+   * wish to use this as the primary key should create a getter method annotated with @Id
+   *
    * <p>This is in the (\w|_){1,80}-\w{1,8} format specified by RFC 5730 for roidType.
    *
    * @see <a href="https://tools.ietf.org/html/rfc5730">RFC 5730</a>
    */
-  @Id
-  // not persisted so that we can store these in references to other objects. Subclasses that wish
-  // to use this as the primary key should create a getter method annotated with @Id
-  @Transient
-  String repoId;
+  @Id @Transient String repoId;
 
-  /** The ID of the registrar that is currently sponsoring this resource. */
+  /**
+   * The ID of the registrar that is currently sponsoring this resource.
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
   @Index
-  @Column(name = "currentSponsorRegistrarId", nullable = false)
+  @Column(name = "currentSponsorRegistrarId")
   String currentSponsorClientId;
 
-  /** The ID of the registrar that created this resource. */
-  @Column(name = "creationRegistrarId", nullable = false)
+  /**
+   * The ID of the registrar that created this resource.
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
+  @Column(name = "creationRegistrarId")
   String creationClientId;
 
   /**
@@ -91,13 +99,17 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   @Column(name = "lastEppUpdateRegistrarId")
   String lastEppUpdateClientId;
 
-  /** The time when this resource was created. */
-  // Map the method to XML, not the field, because if we map the field (with an adaptor class) it
-  // will never be omitted from the xml even if the timestamp inside creationTime is null and we
-  // return null from the adaptor. (Instead it gets written as an empty tag.)
-  @Column(nullable = false)
-  @Index
-  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
+  /**
+   * The time when this resource was created.
+   *
+   * <p>Map the method to XML, not the field, because if we map the field (with an adaptor class) it
+   * will never be omitted from the xml even if the timestamp inside creationTime is null and we
+   * return null from the adaptor (instead it gets written as an empty tag).
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
+  @Index CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.
@@ -112,8 +124,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * out of the index at that time, as long as we query for resources whose deletion time is before
    * now.
    */
-  @Index
-  DateTime deletionTime;
+  @Index DateTime deletionTime;
 
   /**
    * The time that this resource was last updated.
@@ -144,7 +155,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     return repoId;
   }
 
-  // Hibernate needs this to populate the repo ID, but no one else should ever use it
+  /** This method exists solely to satisfy Hibernate. Use {@link Builder} instead. */
   @SuppressWarnings("UnusedMethod")
   private void setRepoId(String repoId) {
     this.repoId = repoId;
@@ -375,7 +386,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   private static LoadingCache<VKey<? extends EppResource>, EppResource> createEppResourcesCache(
       Duration expiry) {
     return CacheBuilder.newBuilder()
-        .expireAfterWrite(expiry.getMillis(), MILLISECONDS)
+        .expireAfterWrite(java.time.Duration.ofMillis(expiry.getMillis()))
         .maximumSize(getEppResourceMaxCachedEntries())
         .build(CACHE_LOADER);
   }

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -61,7 +61,17 @@ public abstract class ImmutableObject implements Cloneable {
   private boolean equalsImmutableObject(ImmutableObject other) {
     return getClass().equals(other.getClass())
         && hashCode() == other.hashCode()
-        && ModelUtils.getFieldValues(this).equals(ModelUtils.getFieldValues(other));
+        && getSignificantFields().equals(other.getSignificantFields());
+  }
+
+  /**
+   * Returns the map of significant fields (fields that we care about for purposes of comparison and
+   * display).
+   *
+   * <p>Isolated into a method so that derived classes can override it.
+   */
+  protected Map<Field, Object> getSignificantFields() {
+    return ModelUtils.getFieldValues(this);
   }
 
   @Override
@@ -72,7 +82,7 @@ public abstract class ImmutableObject implements Cloneable {
   @Override
   public int hashCode() {
     if (hashCode == null) {
-      hashCode = Arrays.hashCode(ModelUtils.getFieldValues(this).values().toArray());
+      hashCode = Arrays.hashCode(getSignificantFields().values().toArray());
     }
     return hashCode;
   }
@@ -111,7 +121,7 @@ public abstract class ImmutableObject implements Cloneable {
   @Override
   public String toString() {
     NavigableMap<String, Object> sortedFields = new TreeMap<>();
-    for (Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
+    for (Entry<Field, Object> entry : getSignificantFields().entrySet()) {
       sortedFields.put(entry.getKey().getName(), entry.getValue());
     }
     return toStringHelper(sortedFields);
@@ -121,7 +131,7 @@ public abstract class ImmutableObject implements Cloneable {
   public String toHydratedString() {
     // We can't use ImmutableSortedMap because we need to allow null values.
     NavigableMap<String, Object> sortedFields = new TreeMap<>();
-    for (Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
+    for (Entry<Field, Object> entry : getSignificantFields().entrySet()) {
       Field field = entry.getKey();
       Object value = entry.getValue();
       sortedFields.put(
@@ -161,7 +171,7 @@ public abstract class ImmutableObject implements Cloneable {
       // LinkedHashMap to preserve field ordering and because ImmutableMap forbids null
       // values.
       Map<String, Object> result = new LinkedHashMap<>();
-      for (Entry<Field, Object> entry : ModelUtils.getFieldValues(o).entrySet()) {
+      for (Entry<Field, Object> entry : ((ImmutableObject) o).getSignificantFields().entrySet()) {
         Field field = entry.getKey();
         if (!field.isAnnotationPresent(IgnoredInDiffableMap.class)) {
           result.put(field.getName(), toMapRecursive(entry.getValue()));

core/src/main/java/google/registry/model/ModelUtils.java

@@ -194,7 +194,7 @@ public class ModelUtils {
    * returned map in its implementation of {@link ImmutableObject#toString} and {@link
    * ImmutableObject#equals}, which work by comparing and printing these maps.
    */
-  static Map<Field, Object> getFieldValues(Object instance) {
+  public static Map<Field, Object> getFieldValues(Object instance) {
     // Don't make this ImmutableMap because field values can be null.
     Map<Field, Object> values = new LinkedHashMap<>();
     for (Field field : getAllFields(instance.getClass()).values()) {

core/src/main/java/google/registry/model/OteAccountBuilder.java

@@ -252,7 +252,8 @@ public final class OteAccountBuilder {
   private void saveAllEntities() {
     tm().assertInTransaction();
 
-    ImmutableList<Registry> registries = ImmutableList.of(sunriseTld, gaTld, eapTld);
+    // use ImmutableObject instead of Registry so that the Key generation doesn't break
+    ImmutableList<ImmutableObject> registries = ImmutableList.of(sunriseTld, gaTld, eapTld);
     ImmutableList<RegistrarContact> contacts = contactsBuilder.build();
 
     if (!replaceExisting) {

core/src/main/java/google/registry/model/OteStats.java

@@ -59,25 +59,11 @@ public class OteStats {
 
   private OteStats() {}
 
-  private static final Predicate<EppInput> HAS_CLAIMS_NOTICE =
-      eppInput -> {
-        Optional<LaunchCreateExtension> launchCreate =
-            eppInput.getSingleExtension(LaunchCreateExtension.class);
-        return launchCreate.isPresent() && launchCreate.get().getNotice() != null;
-      };
-
   private static final Predicate<EppInput> HAS_SEC_DNS =
       eppInput ->
           eppInput.getSingleExtension(SecDnsCreateExtension.class).isPresent()
               || eppInput.getSingleExtension(SecDnsUpdateExtension.class).isPresent();
 
-  private static final Predicate<EppInput> IS_SUNRISE =
-      eppInput -> {
-        Optional<LaunchCreateExtension> launchCreate =
-            eppInput.getSingleExtension(LaunchCreateExtension.class);
-        return launchCreate.isPresent() && !isNullOrEmpty(launchCreate.get().getSignedMarks());
-      };
-
   private static final Predicate<EppInput> IS_IDN =
       eppInput ->
           ((DomainCommand.Create)
@@ -94,6 +80,18 @@ public class OteStats {
                           .getResourceCommand())
                   .getInetAddresses());
 
+  private static boolean hasClaimsNotice(EppInput eppInput) {
+    Optional<LaunchCreateExtension> launchCreate =
+        eppInput.getSingleExtension(LaunchCreateExtension.class);
+    return launchCreate.isPresent() && launchCreate.get().getNotice() != null;
+  }
+
+  private static boolean isSunrise(EppInput eppInput) {
+    Optional<LaunchCreateExtension> launchCreate =
+        eppInput.getSingleExtension(LaunchCreateExtension.class);
+    return launchCreate.isPresent() && !isNullOrEmpty(launchCreate.get().getSignedMarks());
+  }
+
   /** Enum defining the distinct statistics (types of registrar actions) to record. */
   public enum StatType {
     CONTACT_CREATES(0, equalTo(Type.CONTACT_CREATE)),
@@ -107,8 +105,8 @@ public class OteStats {
     DOMAIN_CREATES(0, equalTo(Type.DOMAIN_CREATE)),
     DOMAIN_CREATES_ASCII(1, equalTo(Type.DOMAIN_CREATE), IS_IDN.negate()),
     DOMAIN_CREATES_IDN(1, equalTo(Type.DOMAIN_CREATE), IS_IDN),
-    DOMAIN_CREATES_START_DATE_SUNRISE(1, equalTo(Type.DOMAIN_CREATE), IS_SUNRISE),
-    DOMAIN_CREATES_WITH_CLAIMS_NOTICE(1, equalTo(Type.DOMAIN_CREATE), HAS_CLAIMS_NOTICE),
+    DOMAIN_CREATES_START_DATE_SUNRISE(1, equalTo(Type.DOMAIN_CREATE), OteStats::isSunrise),
+    DOMAIN_CREATES_WITH_CLAIMS_NOTICE(1, equalTo(Type.DOMAIN_CREATE), OteStats::hasClaimsNotice),
     DOMAIN_CREATES_WITH_FEE(
         1,
         equalTo(Type.DOMAIN_CREATE),

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -33,12 +33,14 @@ import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.TimeOfYear;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.GracePeriod;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.token.AllocationToken;
@@ -46,6 +48,7 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -56,16 +59,14 @@ import javax.persistence.Column;
 import javax.persistence.Embedded;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.MappedSuperclass;
+import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
 /** A billable event in a domain's lifecycle. */
 @MappedSuperclass
-@WithLongVKey
 public abstract class BillingEvent extends ImmutableObject
     implements Buildable, TransferServerApproveEntity {
 
@@ -107,10 +108,7 @@ public abstract class BillingEvent extends ImmutableObject
   }
 
   /** Entity id. */
-  @Id
-  @javax.persistence.Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
-  Long id;
+  @Id @javax.persistence.Id Long id;
 
   @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
 
@@ -148,6 +146,21 @@ public abstract class BillingEvent extends ImmutableObject
   @Nullable
   Set<Flag> flags;
 
+  @PostLoad
+  void postLoad() {
+    parent =
+        Key.create(
+            Key.create(DomainBase.class, domainRepoId),
+            HistoryEntry.class,
+            domainHistoryRevisionId);
+  }
+
+  @OnLoad
+  void onLoad() {
+    domainHistoryRevisionId = parent.getId();
+    domainRepoId = parent.getParent().getName();
+  }
+
   public String getClientId() {
     return clientId;
   }
@@ -244,7 +257,6 @@ public abstract class BillingEvent extends ImmutableObject
     }
 
     public B setParent(Key<HistoryEntry> parentKey) {
-      // TODO(shicong): Figure out how to set domainHistoryRevisionId and domainRepoId
       getInstance().parent = parentKey;
       return thisCastToDerived();
     }
@@ -257,6 +269,11 @@ public abstract class BillingEvent extends ImmutableObject
       checkNotNull(instance.eventTime, "Event time must be set");
       checkNotNull(instance.targetId, "Target ID must be set");
       checkNotNull(instance.parent, "Parent must be set");
+      checkNotNull(instance.parent.getParent(), "parent.getParent() must be set");
+      checkNotNull(
+          instance.parent.getParent().getName(), "parent.getParent().getName() must be set");
+      instance.domainHistoryRevisionId = instance.parent.getId();
+      instance.domainRepoId = instance.parent.getParent().getName();
       return super.build();
     }
   }
@@ -274,7 +291,8 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "allocation_token_id")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
-  public static class OneTime extends BillingEvent {
+  @WithLongVKey
+  public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
     @AttributeOverrides({
@@ -309,7 +327,7 @@ public abstract class BillingEvent extends ImmutableObject
      * Cancellation}s.
      */
     @Column(name = "cancellation_matching_billing_recurrence_id")
-    VKey<? extends BillingEvent> cancellationMatchingBillingEvent;
+    VKey<Recurring> cancellationMatchingBillingEvent;
 
     /**
      * The {@link AllocationToken} used in the creation of this event, or null if one was not used.
@@ -391,7 +409,7 @@ public abstract class BillingEvent extends ImmutableObject
       }
 
       public Builder setCancellationMatchingBillingEvent(
-          VKey<? extends BillingEvent> cancellationMatchingBillingEvent) {
+          VKey<Recurring> cancellationMatchingBillingEvent) {
         getInstance().cancellationMatchingBillingEvent = cancellationMatchingBillingEvent;
         return this;
       }
@@ -450,7 +468,8 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
-  public static class Recurring extends BillingEvent {
+  @WithLongVKey
+  public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity {
 
     /**
      * The billing event recurs every year between {@link #eventTime} and this time on the
@@ -544,7 +563,8 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "billingTime")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
-  public static class Cancellation extends BillingEvent {
+  @WithLongVKey
+  public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billing time of the charge that is being cancelled. */
     @Index
@@ -664,7 +684,8 @@ public abstract class BillingEvent extends ImmutableObject
   /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
-  public static class Modification extends BillingEvent {
+  @WithLongVKey
+  public static class Modification extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The change in cost that should be applied to the original billing event. */
     Money cost;

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -14,12 +14,25 @@
 
 package google.registry.model.contact;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
-import google.registry.model.EppResource;
+import com.googlecode.objectify.annotation.EntitySubclass;
+import google.registry.model.ImmutableObject;
+import google.registry.model.contact.ContactHistory.ContactHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
 
 /**
  * A persisted history entry representing an EPP modification to a contact.
@@ -36,21 +49,128 @@ import javax.persistence.Entity;
       @javax.persistence.Index(columnList = "historyType"),
       @javax.persistence.Index(columnList = "historyModificationTime")
     })
-public class ContactHistory extends HistoryEntry {
+@EntitySubclass
+@Access(AccessType.FIELD)
+@IdClass(ContactHistoryId.class)
+public class ContactHistory extends HistoryEntry implements SqlEntity {
+
   // Store ContactBase instead of ContactResource so we don't pick up its @Id
-  ContactBase contactBase;
+  @Nullable ContactBase contactBase;
 
-  @Column(nullable = false)
-  VKey<ContactResource> contactRepoId;
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getContactRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
 
-  /** The state of the {@link ContactBase} object at this point in time. */
-  public ContactBase getContactBase() {
-    return contactBase;
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setContactRepoId(String contactRepoId) {
+    parent = Key.create(ContactResource.class, contactRepoId);
+  }
+
+  @Id
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
+  /**
+   * The values of all the fields on the {@link ContactBase} object after the action represented by
+   * this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<ContactBase> getContactBase() {
+    return Optional.ofNullable(contactBase);
   }
 
   /** The key to the {@link ContactResource} this is based off of. */
-  public VKey<ContactResource> getContactRepoId() {
-    return contactRepoId;
+  public VKey<ContactResource> getParentVKey() {
+    return VKey.create(ContactResource.class, getContactRepoId());
+  }
+
+  /** Creates a {@link VKey} instance for this entity. */
+  public VKey<ContactHistory> createVKey() {
+    return VKey.create(
+        ContactHistory.class, new ContactHistoryId(getContactRepoId(), getId()), Key.create(this));
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Normally Hibernate would see that the contact fields are all null and would fill contactBase
+    // with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+    if (contactBase != null && contactBase.getContactId() == null) {
+      contactBase = null;
+    }
+  }
+
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
+  /** Class to represent the composite primary key of {@link ContactHistory} entity. */
+  static class ContactHistoryId extends ImmutableObject implements Serializable {
+
+    private String contactRepoId;
+
+    private Long id;
+
+    /** Hibernate requires this default constructor. */
+    private ContactHistoryId() {}
+
+    ContactHistoryId(String contactRepoId, long id) {
+      this.contactRepoId = contactRepoId;
+      this.id = id;
+    }
+
+    /**
+     * Returns the contact repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getContactRepoId() {
+      return contactRepoId;
+    }
+
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
+      return id;
+    }
+
+    /**
+     * Sets the contact repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setContactRepoId(String contactRepoId) {
+      this.contactRepoId = contactRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
+      this.id = id;
+    }
   }
 
   @Override
@@ -71,18 +191,8 @@ public class ContactHistory extends HistoryEntry {
       return this;
     }
 
-    public Builder setContactRepoId(VKey<ContactResource> contactRepoId) {
-      getInstance().contactRepoId = contactRepoId;
-      contactRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
-      return this;
-    }
-
-    // We can remove this once all HistoryEntries are converted to History objects
-    @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().contactRepoId =
-          VKey.create(ContactResource.class, parent.getName(), (Key<ContactResource>) parent);
+    public Builder setContactRepoId(String contactRepoId) {
+      getInstance().parent = Key.create(ContactResource.class, contactRepoId);
       return this;
     }
   }

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -51,7 +51,6 @@ public class ContactResource extends ContactBase
 
   @Override
   public VKey<ContactResource> createVKey() {
-    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.create(ContactResource.class, getRepoId(), Key.create(this));
   }
 

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -14,12 +14,12 @@
 
 package google.registry.model.domain;
 
-
 import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
@@ -82,17 +82,57 @@ public class DomainBase extends DomainContent
     return super.nsHosts;
   }
 
+  /**
+   * Returns the set of {@link GracePeriod} associated with the domain.
+   *
+   * <p>This is the getter method specific for Hibernate to access the field so it is set to
+   * private. The caller can use the public {@link #getGracePeriods()} to get the grace periods.
+   *
+   * <p>Note that we need to set `insertable = false, updatable = false` for @JoinColumn, otherwise
+   * Hibernate would try to set the foreign key to null(through an UPDATE TABLE sql) instead of
+   * deleting the whole entry from the table when the {@link GracePeriod} is removed from the set.
+   */
   @Access(AccessType.PROPERTY)
   @OneToMany(
       cascade = {CascadeType.ALL},
       fetch = FetchType.EAGER,
       orphanRemoval = true)
-  @JoinColumn(name = "domainRepoId", referencedColumnName = "repoId")
+  @JoinColumn(
+      name = "domainRepoId",
+      referencedColumnName = "repoId",
+      insertable = false,
+      updatable = false)
   @SuppressWarnings("UnusedMethod")
   private Set<GracePeriod> getInternalGracePeriods() {
     return gracePeriods;
   }
 
+  /**
+   * Returns the set of {@link DelegationSignerData} associated with the domain.
+   *
+   * <p>This is the getter method specific for Hibernate to access the field so it is set to
+   * private. The caller can use the public {@link #getDsData()} to get the DS data.
+   *
+   * <p>Note that we need to set `insertable = false, updatable = false` for @JoinColumn, otherwise
+   * Hibernate would try to set the foreign key to null(through an UPDATE TABLE sql) instead of
+   * deleting the whole entry from the table when the {@link DelegationSignerData} is removed from
+   * the set.
+   */
+  @Access(AccessType.PROPERTY)
+  @OneToMany(
+      cascade = {CascadeType.ALL},
+      fetch = FetchType.EAGER,
+      orphanRemoval = true)
+  @JoinColumn(
+      name = "domainRepoId",
+      referencedColumnName = "repoId",
+      insertable = false,
+      updatable = false)
+  @SuppressWarnings("UnusedMethod")
+  private Set<DelegationSignerData> getInternalDelegationSignerData() {
+    return dsData;
+  }
+
   @Override
   public VKey<DomainBase> createVKey() {
     return VKey.create(DomainBase.class, getRepoId(), Key.create(this));

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -35,11 +35,13 @@ import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static google.registry.util.DomainNameUtils.getTldFromDomainName;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Ordering;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Streams;
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
@@ -49,6 +51,7 @@ import google.registry.flows.ResourceFlowUtils;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.launch.LaunchNotice;
 import google.registry.model.domain.rgp.GracePeriodStatus;
@@ -57,6 +60,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registry.Registry;
+import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
@@ -65,7 +69,6 @@ import java.util.HashSet;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.function.Predicate;
 import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -209,6 +212,15 @@ public class DomainContent extends EppResource
   @Column(name = "deletion_poll_message_id")
   VKey<PollMessage.OneTime> deletePollMessage;
 
+  /**
+   * History record for the delete poll message.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Column(name = "deletion_poll_message_history_id")
+  @Ignore
+  Long deletePollMessageHistoryId;
+
   /**
    * The recurring billing event associated with this domain's autorenewals.
    *
@@ -220,6 +232,15 @@ public class DomainContent extends EppResource
   @Column(name = "billing_recurrence_id")
   VKey<BillingEvent.Recurring> autorenewBillingEvent;
 
+  /**
+   * History record for the autorenew billing event.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Column(name = "billing_recurrence_history_id")
+  @Ignore
+  Long autorenewBillingEventHistoryId;
+
   /**
    * The recurring poll message associated with this domain's autorenewals.
    *
@@ -231,6 +252,13 @@ public class DomainContent extends EppResource
   @Column(name = "autorenew_poll_message_id")
   VKey<PollMessage.Autorenew> autorenewPollMessage;
 
+  /**
+   * History record for the autorenew poll message.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Ignore Long autorenewPollMessageHistoryId;
+
   /** The unexpired grace periods for this domain (some of which may not be active yet). */
   @Transient Set<GracePeriod> gracePeriods;
 
@@ -275,21 +303,28 @@ public class DomainContent extends EppResource
         allContacts.stream().map(DesignatedContact::reconstitute).collect(toImmutableSet());
     setContactFields(allContacts, true);
 
-    // We have to return the cloned object here because the original object's
-    // hashcode is not correct due to the change to its domainRepoId. The cloned
-    // object will have a null hashcode so that it can get a recalculated hashcode
-    // when its hashCode() is invoked.
+    // We have to return the cloned object here because the original object's hashcode is not
+    // correct due to the change to its domainRepoId and history ids. The cloned object will have a
+    // null hashcode so that it can get a recalculated hashcode when its hashCode() is invoked.
     // TODO(b/162739503): Remove this after fully migrating to Cloud SQL.
-    if (gracePeriods != null) {
-      gracePeriods =
-          gracePeriods.stream()
-              .map(gracePeriod -> gracePeriod.cloneWithDomainRepoId(getRepoId()))
-              .collect(toImmutableSet());
-    }
+    gracePeriods =
+        nullToEmptyImmutableCopy(gracePeriods).stream()
+            .map(gracePeriod -> gracePeriod.cloneAfterOfyLoad(getRepoId()))
+            .collect(toImmutableSet());
+
+    // Restore history record ids.
+    autorenewPollMessageHistoryId = getHistoryId(autorenewPollMessage);
+    autorenewBillingEventHistoryId = getHistoryId(autorenewBillingEvent);
+    deletePollMessageHistoryId = getHistoryId(deletePollMessage);
+    dsData =
+        nullToEmptyImmutableCopy(dsData).stream()
+            .map(dsData -> dsData.cloneWithDomainRepoId(getRepoId()))
+            .collect(toImmutableSet());
   }
 
   @PostLoad
-  void postLoad() {
+  @SuppressWarnings("UnusedMethod")
+  private final void postLoad() {
     // Reconstitute the contact list.
     ImmutableSet.Builder<DesignatedContact> contactsBuilder = new ImmutableSet.Builder<>();
 
@@ -308,6 +343,27 @@ public class DomainContent extends EppResource
     }
 
     allContacts = contactsBuilder.build();
+
+    // Reconstitute the composite ofy keys from the SQL data.
+    Key<DomainBase> myKey = Key.create(DomainBase.class, getRepoId());
+    deletePollMessage = restoreOfyFrom(myKey, deletePollMessage, deletePollMessageHistoryId);
+    autorenewBillingEvent =
+        restoreOfyFrom(myKey, autorenewBillingEvent, autorenewBillingEventHistoryId);
+    autorenewPollMessage =
+        restoreOfyFrom(myKey, autorenewPollMessage, autorenewPollMessageHistoryId);
+
+    if (transferData != null) {
+      transferData.restoreOfyKeys(myKey);
+    }
+  }
+
+  public static <T> VKey<T> restoreOfyFrom(Key<DomainBase> domainKey, VKey<T> key, Long historyId) {
+    if (historyId == null) {
+      // This is a legacy key (or a null key, in which case this works too)
+      return VKey.restoreOfyFrom(key, EntityGroupRoot.class, "per-tld");
+    } else {
+      return VKey.restoreOfyFrom(key, domainKey, HistoryEntry.class, historyId);
+    }
   }
 
   public ImmutableSet<String> getSubordinateHosts() {
@@ -380,7 +436,7 @@ public class DomainContent extends EppResource
   // Hibernate needs this in order to populate nsHosts but no one else should ever use it
   @SuppressWarnings("UnusedMethod")
   private void setNsHosts(Set<VKey<HostResource>> nsHosts) {
-    this.nsHosts = nsHosts;
+    this.nsHosts = forceEmptyToNull(nsHosts);
   }
 
   // Hibernate needs this in order to populate gracePeriods but no one else should ever use it
@@ -389,6 +445,12 @@ public class DomainContent extends EppResource
     this.gracePeriods = gracePeriods;
   }
 
+  // Hibernate needs this in order to populate dsData but no one else should ever use it
+  @SuppressWarnings("UnusedMethod")
+  private void setInternalDelegationSignerData(Set<DelegationSignerData> dsData) {
+    this.dsData = dsData;
+  }
+
   public final String getCurrentSponsorClientId() {
     return getPersistedCurrentSponsorClientId();
   }
@@ -657,13 +719,31 @@ public class DomainContent extends EppResource
             + " use DomainBase instead");
   }
 
+  /**
+   * Obtains a history id from the given key.
+   *
+   * <p>The key must be a composite key either of the form domain-key/history-key/long-event-key or
+   * EntityGroupRoot/long-event-key (for legacy keys). In the latter case or for a null key returns
+   * a history id of null.
+   */
+  public static Long getHistoryId(VKey<?> key) {
+    if (key == null) {
+      return null;
+    }
+    Key<?> parent = key.getOfyKey().getParent();
+    if (parent == null || parent.getKind().equals("EntityGroupRoot")) {
+      return null;
+    }
+    return parent.getId();
+  }
+
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
   static final Predicate<DesignatedContact> IS_REGISTRANT =
       (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);
 
   /** An override of {@link EppResource#asBuilder} with tighter typing. */
   @Override
-  public Builder asBuilder() {
+  public Builder<? extends DomainContent, ?> asBuilder() {
     return new Builder<>(clone(this));
   }
 
@@ -698,7 +778,19 @@ public class DomainContent extends EppResource
       }
       checkArgumentNotNull(instance.getRegistrant(), "Missing registrant");
       instance.tld = getTldFromDomainName(instance.fullyQualifiedDomainName);
-      return super.build();
+
+      T newDomain = super.build();
+      // Hibernate throws exception if gracePeriods or dsData is null because we enabled all
+      // cascadable operations and orphan removal.
+      newDomain.gracePeriods =
+          newDomain.gracePeriods == null ? ImmutableSet.of() : newDomain.gracePeriods;
+      newDomain.dsData =
+          newDomain.dsData == null
+              ? ImmutableSet.of()
+              : newDomain.dsData.stream()
+                  .map(ds -> ds.cloneWithDomainRepoId(instance.getRepoId()))
+                  .collect(toImmutableSet());
+      return newDomain;
     }
 
     public B setDomainName(String domainName) {
@@ -820,16 +912,19 @@ public class DomainContent extends EppResource
 
     public B setDeletePollMessage(VKey<PollMessage.OneTime> deletePollMessage) {
       getInstance().deletePollMessage = deletePollMessage;
+      getInstance().deletePollMessageHistoryId = getHistoryId(deletePollMessage);
       return thisCastToDerived();
     }
 
     public B setAutorenewBillingEvent(VKey<BillingEvent.Recurring> autorenewBillingEvent) {
       getInstance().autorenewBillingEvent = autorenewBillingEvent;
+      getInstance().autorenewBillingEventHistoryId = getHistoryId(autorenewBillingEvent);
       return thisCastToDerived();
     }
 
     public B setAutorenewPollMessage(VKey<PollMessage.Autorenew> autorenewPollMessage) {
       getInstance().autorenewPollMessage = autorenewPollMessage;
+      getInstance().autorenewPollMessageHistoryId = getHistoryId(autorenewPollMessage);
       return thisCastToDerived();
     }
 

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -14,19 +14,45 @@
 
 package google.registry.model.domain;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
-import google.registry.model.EppResource;
-import google.registry.model.contact.ContactResource;
+import com.googlecode.objectify.annotation.EntitySubclass;
+import com.googlecode.objectify.annotation.Ignore;
+import google.registry.model.ImmutableObject;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
 import google.registry.model.host.HostResource;
+import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import java.io.Serializable;
+import java.util.Optional;
 import java.util.Set;
+import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Index;
+import javax.persistence.JoinColumn;
+import javax.persistence.JoinColumns;
 import javax.persistence.JoinTable;
+import javax.persistence.OneToMany;
+import javax.persistence.PostLoad;
+import javax.persistence.Table;
 
 /**
  * A persisted history entry representing an EPP modification to a domain.
@@ -36,43 +62,232 @@ import javax.persistence.JoinTable;
  * the foreign-keyed fields in that class can refer to this object.
  */
 @Entity
-@javax.persistence.Table(
+@Table(
     indexes = {
-      @javax.persistence.Index(columnList = "creationTime"),
-      @javax.persistence.Index(columnList = "historyRegistrarId"),
-      @javax.persistence.Index(columnList = "historyType"),
-      @javax.persistence.Index(columnList = "historyModificationTime")
+      @Index(columnList = "creationTime"),
+      @Index(columnList = "historyRegistrarId"),
+      @Index(columnList = "historyType"),
+      @Index(columnList = "historyModificationTime")
     })
-public class DomainHistory extends HistoryEntry {
+@EntitySubclass
+@Access(AccessType.FIELD)
+@IdClass(DomainHistoryId.class)
+public class DomainHistory extends HistoryEntry implements SqlEntity {
+
   // Store DomainContent instead of DomainBase so we don't pick up its @Id
-  DomainContent domainContent;
+  @Nullable DomainContent domainContent;
 
-  @Column(nullable = false)
-  VKey<DomainBase> domainRepoId;
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
 
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setDomainRepoId(String domainRepoId) {
+    parent = Key.create(DomainBase.class, domainRepoId);
+  }
+
+  // We could have reused domainContent.nsHosts here, but Hibernate throws a weird exception after
+  // we change to use a composite primary key.
+  // TODO(b/166776754): Investigate if we can reuse domainContent.nsHosts for storing host keys.
+  @Ignore
   @ElementCollection
   @JoinTable(name = "DomainHistoryHost")
-  @Access(AccessType.PROPERTY)
   @Column(name = "host_repo_id")
+  Set<VKey<HostResource>> nsHosts;
+
+  @OneToMany(
+      cascade = {CascadeType.ALL},
+      fetch = FetchType.EAGER,
+      orphanRemoval = true)
+  @JoinColumns({
+    @JoinColumn(
+        name = "domainHistoryRevisionId",
+        referencedColumnName = "historyRevisionId",
+        insertable = false,
+        updatable = false),
+    @JoinColumn(
+        name = "domainRepoId",
+        referencedColumnName = "domainRepoId",
+        insertable = false,
+        updatable = false)
+  })
+  Set<DomainDsDataHistory> dsDataHistories;
+
+  @Override
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @AttributeOverrides({
+      @AttributeOverride(
+          name = "unit",
+          column = @Column(name = "historyPeriodUnit")),
+      @AttributeOverride(
+          name = "value",
+          column = @Column(name = "historyPeriodValue"))
+  })
+  public Period getPeriod() {
+    return super.getPeriod();
+  }
+
+  /**
+   * For transfers, the id of the other registrar.
+   *
+   * <p>For requests and cancels, the other registrar is the losing party (because the registrar
+   * sending the EPP transfer command is the gaining party). For approves and rejects, the other
+   * registrar is the gaining party.
+   */
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @Column(name = "historyOtherRegistrarId")
+  public String getOtherRegistrarId() {
+    return super.getOtherClientId();
+  }
+
+  /**
+   * Logging field for transaction reporting.
+   *
+   * <p>This will be empty for any DomainHistory/HistoryEntry generated before this field was added,
+   * mid-2017, as well as any action that does not generate billable events (e.g. updates).
+   *
+   * <p>This method is dedicated for Hibernate, external caller should use {@link
+   * #getDomainTransactionRecords()}.
+   */
+  @Access(AccessType.PROPERTY)
+  @OneToMany(cascade = {CascadeType.ALL})
+  @JoinColumn(name = "historyRevisionId", referencedColumnName = "historyRevisionId")
+  @JoinColumn(name = "domainRepoId", referencedColumnName = "domainRepoId")
+  @SuppressWarnings("unused")
+  private Set<DomainTransactionRecord> getInternalDomainTransactionRecords() {
+    return domainTransactionRecords;
+  }
+
+  /** Sets the domain transaction records. This method is dedicated for Hibernate. */
+  @SuppressWarnings("unused")
+  private void setInternalDomainTransactionRecords(
+      Set<DomainTransactionRecord> domainTransactionRecords) {
+    this.domainTransactionRecords = domainTransactionRecords;
+  }
+
+  @Id
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
+  /** Returns keys to the {@link HostResource} that are the nameservers for the domain. */
   public Set<VKey<HostResource>> getNsHosts() {
-    return domainContent.nsHosts;
+    return nsHosts;
   }
 
-  /** The state of the {@link DomainContent} object at this point in time. */
-  public DomainContent getDomainContent() {
-    return domainContent;
+  /** Returns the collection of {@link DomainDsDataHistory} instances. */
+  public ImmutableSet<DomainDsDataHistory> getDsDataHistories() {
+    return nullToEmptyImmutableCopy(dsDataHistories);
   }
 
-  /** The key to the {@link ContactResource} this is based off of. */
-  public VKey<DomainBase> getDomainRepoId() {
-    return domainRepoId;
+  /**
+   * The values of all the fields on the {@link DomainContent} object after the action represented
+   * by this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<DomainContent> getDomainContent() {
+    return Optional.ofNullable(domainContent);
   }
 
-  // Hibernate needs this in order to populate nsHosts but no one else should ever use it
-  @SuppressWarnings("UnusedMethod")
-  private void setNsHosts(Set<VKey<HostResource>> nsHosts) {
+  /** The key to the {@link DomainBase} this is based off of. */
+  public VKey<DomainBase> getParentVKey() {
+    return VKey.create(DomainBase.class, getDomainRepoId());
+  }
+
+  /** Creates a {@link VKey} instance for this entity. */
+  public VKey<DomainHistory> createVKey() {
+    return VKey.create(
+        DomainHistory.class, new DomainHistoryId(getDomainRepoId(), getId()), Key.create(this));
+  }
+
+  @PostLoad
+  void postLoad() {
     if (domainContent != null) {
-      domainContent.nsHosts = nsHosts;
+      domainContent.nsHosts = nullToEmptyImmutableCopy(nsHosts);
+      // Normally Hibernate would see that the domain fields are all null and would fill
+      // domainContent with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+      if (domainContent.getDomainName() == null) {
+        domainContent = null;
+      } else {
+        if (domainContent.getRepoId() == null) {
+          domainContent = domainContent.asBuilder().setRepoId(parent.getName()).build();
+        }
+      }
+    }
+  }
+
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
+  /** Class to represent the composite primary key of {@link DomainHistory} entity. */
+  static class DomainHistoryId extends ImmutableObject implements Serializable {
+
+    private String domainRepoId;
+
+    private Long id;
+
+    /** Hibernate requires this default constructor. */
+    private DomainHistoryId() {}
+
+    DomainHistoryId(String domainRepoId, long id) {
+      this.domainRepoId = domainRepoId;
+      this.id = id;
+    }
+
+    /**
+     * Returns the domain repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getDomainRepoId() {
+      return domainRepoId;
+    }
+
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
+      return id;
+    }
+
+    /**
+     * Sets the domain repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setDomainRepoId(String domainRepoId) {
+      this.domainRepoId = domainRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
+      this.id = id;
     }
   }
 
@@ -94,19 +309,26 @@ public class DomainHistory extends HistoryEntry {
       return this;
     }
 
-    public Builder setDomainRepoId(VKey<DomainBase> domainRepoId) {
-      getInstance().domainRepoId = domainRepoId;
-      domainRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
+    public Builder setDomainRepoId(String domainRepoId) {
+      getInstance().parent = Key.create(DomainBase.class, domainRepoId);
       return this;
     }
 
-    // We can remove this once all HistoryEntries are converted to History objects
     @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().domainRepoId =
-          VKey.create(DomainBase.class, parent.getName(), (Key<DomainBase>) parent);
-      return this;
+    public DomainHistory build() {
+      DomainHistory instance = super.build();
+      // TODO(b/171990736): Assert instance.domainContent is not null after database migration.
+      // Note that we cannot assert that instance.domainContent is not null here because this
+      // builder is also used to convert legacy HistoryEntry objects to DomainHistory, when
+      // domainContent is not available.
+      if (instance.domainContent != null) {
+        instance.nsHosts = nullToEmptyImmutableCopy(instance.domainContent.nsHosts);
+        instance.dsDataHistories =
+            nullToEmptyImmutableCopy(instance.domainContent.getDsData()).stream()
+                .map(dsData -> DomainDsDataHistory.createFrom(instance.id, dsData))
+                .collect(toImmutableSet());
+      }
+      return instance;
     }
   }
 }

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -21,7 +21,9 @@ import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.model.ofy.ObjectifyService;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import javax.annotation.Nullable;
 import javax.persistence.Entity;
 import javax.persistence.Index;
@@ -37,7 +39,7 @@ import org.joda.time.DateTime;
 @Embed
 @Entity
 @Table(indexes = @Index(columnList = "domainRepoId"))
-public class GracePeriod extends GracePeriodBase {
+public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntity {
 
   private static GracePeriod createInternal(
       GracePeriodStatus type,
@@ -52,12 +54,15 @@ public class GracePeriod extends GracePeriodBase {
         (billingEventRecurring != null) == GracePeriodStatus.AUTO_RENEW.equals(type),
         "Recurring billing events must be present on (and only on) autorenew grace periods");
     GracePeriod instance = new GracePeriod();
+    instance.id = ObjectifyService.allocateId();
     instance.type = checkArgumentNotNull(type);
     instance.domainRepoId = checkArgumentNotNull(domainRepoId);
     instance.expirationTime = checkArgumentNotNull(expirationTime);
     instance.clientId = checkArgumentNotNull(clientId);
     instance.billingEventOneTime = billingEventOneTime;
+    instance.billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
     instance.billingEventRecurring = billingEventRecurring;
+    instance.billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
     return instance;
   }
 
@@ -107,14 +112,16 @@ public class GracePeriod extends GracePeriodBase {
   }
 
   /**
-   * Returns a clone of this {@link GracePeriod} with {@link #domainRepoId} set to the given value.
+   * Returns a clone of this {@link GracePeriod} with {@link #domainRepoId} set to the given value
+   * and reconstructed history ids.
    *
    * <p>TODO(b/162739503): Remove this function after fully migrating to Cloud SQL.
    */
-  public GracePeriod cloneWithDomainRepoId(String domainRepoId) {
+  public GracePeriod cloneAfterOfyLoad(String domainRepoId) {
     GracePeriod clone = clone(this);
+    clone.id = ObjectifyService.allocateId();
     clone.domainRepoId = checkArgumentNotNull(domainRepoId);
+    clone.restoreHistoryIds();
     return clone;
   }
-
 }

core/src/main/java/google/registry/model/domain/GracePeriodBase.java

@@ -14,18 +14,21 @@
 
 package google.registry.model.domain;
 
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.ModelUtils;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.persistence.VKey;
+import java.lang.reflect.Field;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.MappedSuperclass;
 import org.joda.time.DateTime;
 
@@ -36,7 +39,6 @@ public class GracePeriodBase extends ImmutableObject {
 
   /** Unique id required for hibernate representation. */
   @javax.persistence.Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
   @Ignore
   Long id;
 
@@ -67,6 +69,10 @@ public class GracePeriodBase extends ImmutableObject {
   @Column(name = "billing_event_id")
   VKey<OneTime> billingEventOneTime = null;
 
+  @Ignore
+  @Column(name = "billing_event_history_id")
+  Long billingEventOneTimeHistoryId;
+
   /**
    * The recurring billing event corresponding to the action that triggered this grace period, if
    * applicable - i.e. if the action was an autorenew - or null in all other cases.
@@ -75,6 +81,14 @@ public class GracePeriodBase extends ImmutableObject {
   @Column(name = "billing_recurrence_id")
   VKey<BillingEvent.Recurring> billingEventRecurring = null;
 
+  @Ignore
+  @Column(name = "billing_recurrence_history_id")
+  Long billingEventRecurringHistoryId;
+
+  public long getId() {
+    return id;
+  }
+
   public GracePeriodStatus getType() {
     return type;
   }
@@ -101,6 +115,7 @@ public class GracePeriodBase extends ImmutableObject {
    * period is not AUTO_RENEW.
    */
   public VKey<BillingEvent.OneTime> getOneTimeBillingEvent() {
+    restoreOfyKeys();
     return billingEventOneTime;
   }
 
@@ -109,6 +124,63 @@ public class GracePeriodBase extends ImmutableObject {
    * period is AUTO_RENEW.
    */
   public VKey<BillingEvent.Recurring> getRecurringBillingEvent() {
+    restoreOfyKeys();
     return billingEventRecurring;
   }
+
+  /**
+   * Restores history ids for composite VKeys after a load from datastore.
+   *
+   * <p>For use by DomainContent.load() ONLY.
+   */
+  protected void restoreHistoryIds() {
+    billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
+    billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
+  }
+
+  /**
+   * Override {@link ImmutableObject#getSignificantFields()} to exclude "id", which breaks equality
+   * testing in the unit tests.
+   */
+  @Override
+  protected Map<Field, Object> getSignificantFields() {
+    restoreOfyKeys();
+    // Can't use streams or ImmutableMap because we can have null values.
+    Map<Field, Object> result = new LinkedHashMap();
+    for (Map.Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
+      if (!entry.getKey().getName().equals("id")) {
+        result.put(entry.getKey(), entry.getValue());
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Restores Ofy keys in the billing events.
+   *
+   * <p>This must be called by all methods that access the one time or recurring billing event keys.
+   * When the billing event keys are loaded from SQL, they are loaded as asymmetric keys because the
+   * database columns that we load them from do not contain all of the information necessary to
+   * reconsitute the Ofy side of the key. In other cases, we restore the Ofy key during the
+   * hibernate {@link javax.persistence.PostLoad} method from the other fields of the object, but we
+   * have been unable to make this work with hibernate's internal persistence model in this case
+   * because the {@link GracePeriod}'s hash code is evaluated prior to these calls, and would be
+   * invalidated by changing the fields.
+   */
+  private final synchronized void restoreOfyKeys() {
+    if (billingEventOneTime != null && !billingEventOneTime.maybeGetOfyKey().isPresent()) {
+      billingEventOneTime =
+          DomainBase.restoreOfyFrom(
+              Key.create(DomainBase.class, domainRepoId),
+              billingEventOneTime,
+              billingEventOneTimeHistoryId);
+    }
+    if (billingEventRecurring != null && !billingEventRecurring.maybeGetOfyKey().isPresent()) {
+      billingEventRecurring =
+          DomainBase.restoreOfyFrom(
+              Key.create(DomainBase.class, domainRepoId),
+              billingEventRecurring,
+              billingEventRecurringHistoryId);
+    }
+  }
 }

core/src/main/java/google/registry/model/domain/Period.java

@@ -31,9 +31,9 @@ public class Period extends ImmutableObject {
   @XmlAttribute
   Unit unit;
 
-  @XmlValue
-  Integer value;
+  @XmlValue Integer value;
 
+  @Enumerated(EnumType.STRING)
   public Unit getUnit() {
     return unit;
   }
@@ -42,6 +42,18 @@ public class Period extends ImmutableObject {
     return value;
   }
 
+  /** This method exists solely to satisfy Hibernate. Use {@link #create(int, Unit)} instead. */
+  @SuppressWarnings("UnusedMethod")
+  private void setUnit(Unit unit) {
+    this.unit = unit;
+  }
+
+  /** This method exists solely to satisfy Hibernate. Use {@link #create(int, Unit)} instead. */
+  @SuppressWarnings("UnusedMethod")
+  private void setValue(Integer value) {
+    this.value = value;
+  }
+
   /** The unit enum. */
   public enum Unit {
     @XmlEnumValue("y")

core/src/main/java/google/registry/model/domain/secdns/DelegationSignerData.java

@@ -14,96 +14,235 @@
 
 package google.registry.model.domain.secdns;
 
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.domain.secdns.DelegationSignerData.DomainDsDataId;
+import java.io.Serializable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Index;
+import javax.persistence.Table;
 import javax.xml.bind.DatatypeConverter;
-import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlType;
-import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
-import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 /**
  * Holds the data necessary to construct a single Delegation Signer (DS) record for a domain.
  *
  * @see <a href="http://tools.ietf.org/html/rfc5910">RFC 5910</a>
  * @see <a href="http://tools.ietf.org/html/rfc4034">RFC 4034</a>
+ *     <p>TODO(shicong): Rename this class to DomainDsData.
  */
 @Embed
 @XmlType(name = "dsData")
-@javax.persistence.Entity
-public class DelegationSignerData extends ImmutableObject {
+@Entity
+@IdClass(DomainDsDataId.class)
+@Table(indexes = @Index(columnList = "domainRepoId"))
+public class DelegationSignerData extends DomainDsDataBase {
 
   private DelegationSignerData() {}
 
-  /** The identifier for this particular key in the domain. */
-  @javax.persistence.Id int keyTag;
-
-  /**
-   * The algorithm used by this key.
-   *
-   * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.1">RFC 4034 Appendix A.1</a>
-   */
-  @XmlElement(name = "alg")
-  int algorithm;
-
-  /**
-   * The algorithm used to generate the digest.
-   *
-   * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.2">RFC 4034 Appendix A.2</a>
-   */
-  int digestType;
-
-  /**
-   * The hexBinary digest of the public key.
-   *
-   * @see <a href="http://tools.ietf.org/html/rfc4034#section-5.1.4">RFC 4034 Section 5.1.4</a>
-   */
-  @XmlJavaTypeAdapter(HexBinaryAdapter.class)
-  byte[] digest;
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    return super.getDomainRepoId();
+  }
 
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
   public int getKeyTag() {
-    return keyTag;
+    return super.getKeyTag();
   }
 
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
   public int getAlgorithm() {
-    return algorithm;
+    return super.getAlgorithm();
   }
 
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
   public int getDigestType() {
-    return digestType;
+    return super.getDigestType();
   }
 
+  @Override
+  @Id
+  @Access(AccessType.PROPERTY)
   public byte[] getDigest() {
-    return digest;
+    return super.getDigest();
   }
 
-  public String getDigestAsString() {
-    return digest == null ? "" : DatatypeConverter.printHexBinary(digest);
+  public DelegationSignerData cloneWithDomainRepoId(String domainRepoId) {
+    DelegationSignerData clone = clone(this);
+    clone.domainRepoId = checkArgumentNotNull(domainRepoId);
+    return clone;
+  }
+
+  public DelegationSignerData cloneWithoutDomainRepoId() {
+    DelegationSignerData clone = clone(this);
+    clone.domainRepoId = null;
+    return clone;
   }
 
   public static DelegationSignerData create(
-      int keyTag, int algorithm, int digestType, byte[] digest) {
+      int keyTag, int algorithm, int digestType, byte[] digest, String domainRepoId) {
     DelegationSignerData instance = new DelegationSignerData();
     instance.keyTag = keyTag;
     instance.algorithm = algorithm;
     instance.digestType = digestType;
     instance.digest = digest;
+    instance.domainRepoId = domainRepoId;
     return instance;
   }
 
+  public static DelegationSignerData create(
+      int keyTag, int algorithm, int digestType, byte[] digest) {
+    return create(keyTag, algorithm, digestType, digest, null);
+  }
+
   public static DelegationSignerData create(
       int keyTag, int algorithm, int digestType, String digestAsHex) {
     return create(keyTag, algorithm, digestType, DatatypeConverter.parseHexBinary(digestAsHex));
   }
 
-  /**
-   * Returns the presentation format of this DS record.
-   *
-   * @see <a href="https://tools.ietf.org/html/rfc4034#section-5.3">RFC 4034 Section 5.3</a>
-   */
-  public String toRrData() {
-    return String.format(
-        "%d %d %d %s",
-        this.keyTag, this.algorithm, this.digestType, DatatypeConverter.printHexBinary(digest));
+  /** Class to represent the composite primary key of {@link DelegationSignerData} entity. */
+  static class DomainDsDataId extends ImmutableObject implements Serializable {
+
+    String domainRepoId;
+
+    int keyTag;
+
+    int algorithm;
+
+    int digestType;
+
+    byte[] digest;
+
+    /** Hibernate requires this default constructor. */
+    private DomainDsDataId() {}
+
+    /** Constructs a {link DomainDsDataId} instance. */
+    DomainDsDataId(String domainRepoId, int keyTag, int algorithm, int digestType, byte[] digest) {
+      this.domainRepoId = domainRepoId;
+      this.keyTag = keyTag;
+      this.algorithm = algorithm;
+      this.digestType = digestType;
+      this.digest = digest;
+    }
+
+    /**
+     * Returns the domain repository ID.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getDomainRepoId() {
+      return domainRepoId;
+    }
+
+    /**
+     * Returns the key tag.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private int getKeyTag() {
+      return keyTag;
+    }
+
+    /**
+     * Returns the algorithm.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private int getAlgorithm() {
+      return algorithm;
+    }
+
+    /**
+     * Returns the digest type.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private int getDigestType() {
+      return digestType;
+    }
+
+    /**
+     * Returns the digest.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private byte[] getDigest() {
+      return digest;
+    }
+
+    /**
+     * Sets the domain repository ID.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setDomainRepoId(String domainRepoId) {
+      this.domainRepoId = domainRepoId;
+    }
+
+    /**
+     * Sets the key tag.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setKeyTag(int keyTag) {
+      this.keyTag = keyTag;
+    }
+
+    /**
+     * Sets the algorithm.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setAlgorithm(int algorithm) {
+      this.algorithm = algorithm;
+    }
+
+    /**
+     * Sets the digest type.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setDigestType(int digestType) {
+      this.digestType = digestType;
+    }
+
+    /**
+     * Sets the digest.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private void setDigest(byte[] digest) {
+      this.digest = digest;
+    }
+
+    public static DomainDsDataId create(
+        String domainRepoId, int keyTag, int algorithm, int digestType, byte[] digest) {
+      return new DomainDsDataId(
+          domainRepoId, keyTag, algorithm, digestType, checkArgumentNotNull(digest));
+    }
   }
 }

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataBase.java

@@ -0,0 +1,151 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.domain.secdns;
+
+import com.googlecode.objectify.annotation.Embed;
+import com.googlecode.objectify.annotation.Ignore;
+import google.registry.model.ImmutableObject;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.Transient;
+import javax.xml.bind.DatatypeConverter;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+/** Base class for {@link DelegationSignerData} and {@link DomainDsDataHistory}. */
+@Embed
+@MappedSuperclass
+@Access(AccessType.FIELD)
+public abstract class DomainDsDataBase extends ImmutableObject implements DatastoreAndSqlEntity {
+
+  @Ignore @XmlTransient @Transient String domainRepoId;
+
+  /** The identifier for this particular key in the domain. */
+  @Transient int keyTag;
+
+  /**
+   * The algorithm used by this key.
+   *
+   * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.1">RFC 4034 Appendix A.1</a>
+   */
+  @Transient
+  @XmlElement(name = "alg")
+  int algorithm;
+
+  /**
+   * The algorithm used to generate the digest.
+   *
+   * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.2">RFC 4034 Appendix A.2</a>
+   */
+  @Transient int digestType;
+
+  /**
+   * The hexBinary digest of the public key.
+   *
+   * @see <a href="http://tools.ietf.org/html/rfc4034#section-5.1.4">RFC 4034 Section 5.1.4</a>
+   */
+  @Transient
+  @XmlJavaTypeAdapter(HexBinaryAdapter.class)
+  byte[] digest;
+
+  public String getDomainRepoId() {
+    return domainRepoId;
+  }
+
+  public int getKeyTag() {
+    return keyTag;
+  }
+
+  public int getAlgorithm() {
+    return algorithm;
+  }
+
+  public int getDigestType() {
+    return digestType;
+  }
+
+  public byte[] getDigest() {
+    return digest;
+  }
+
+  /**
+   * Sets the domain repository ID.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setDomainRepoId(String domainRepoId) {
+    this.domainRepoId = domainRepoId;
+  }
+
+  /**
+   * Sets the key tag.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setKeyTag(int keyTag) {
+    this.keyTag = keyTag;
+  }
+
+  /**
+   * Sets the algorithm.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setAlgorithm(int algorithm) {
+    this.algorithm = algorithm;
+  }
+
+  /**
+   * Sets the digest type.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setDigestType(int digestType) {
+    this.digestType = digestType;
+  }
+
+  /**
+   * Sets the digest.
+   *
+   * <p>This method is private because it is only used by Hibernate.
+   */
+  @SuppressWarnings("unused")
+  private void setDigest(byte[] digest) {
+    this.digest = digest;
+  }
+
+  public String getDigestAsString() {
+    return digest == null ? "" : DatatypeConverter.printHexBinary(digest);
+  }
+
+  /**
+   * Returns the presentation format of this DS record.
+   *
+   * @see <a href="https://tools.ietf.org/html/rfc4034#section-5.3">RFC 4034 Section 5.3</a>
+   */
+  public String toRrData() {
+    return String.format(
+        "%d %d %d %s",
+        this.keyTag, this.algorithm, this.digestType, DatatypeConverter.printHexBinary(digest));
+  }
+}

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java

@@ -0,0 +1,84 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.domain.secdns;
+
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.ofy.ObjectifyService;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+
+/** Entity class to represent a historic {@link DelegationSignerData}. */
+@Entity
+public class DomainDsDataHistory extends DomainDsDataBase {
+
+  @Id Long dsDataHistoryRevisionId;
+
+  /** ID of the {@link DomainHistory} entity that this entity is associated with. */
+  @Column(nullable = false)
+  Long domainHistoryRevisionId;
+
+  private DomainDsDataHistory() {}
+
+  /**
+   * Creates a {@link DomainDsDataHistory} instance from given {@link #domainHistoryRevisionId} and
+   * {@link DelegationSignerData} instance.
+   */
+  public static DomainDsDataHistory createFrom(
+      long domainHistoryRevisionId, DelegationSignerData dsData) {
+    DomainDsDataHistory instance = new DomainDsDataHistory();
+    instance.domainHistoryRevisionId = domainHistoryRevisionId;
+    instance.domainRepoId = dsData.domainRepoId;
+    instance.keyTag = dsData.getKeyTag();
+    instance.algorithm = dsData.getAlgorithm();
+    instance.digestType = dsData.getDigestType();
+    instance.digest = dsData.getDigest();
+    instance.dsDataHistoryRevisionId = ObjectifyService.allocateId();
+    return instance;
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    return super.getDomainRepoId();
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  public int getKeyTag() {
+    return super.getKeyTag();
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  public int getAlgorithm() {
+    return super.getAlgorithm();
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  public int getDigestType() {
+    return super.getDigestType();
+  }
+
+  @Override
+  @Access(AccessType.PROPERTY)
+  @Column(nullable = false)
+  public byte[] getDigest() {
+    return super.getDigest();
+  }
+}

core/src/main/java/google/registry/model/domain/superuser/DomainDeleteSuperuserExtension.java

@@ -20,6 +20,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 /** A superuser extension that may be present on domain delete commands. */
 @XmlRootElement(name = "domainDelete")
 public class DomainDeleteSuperuserExtension extends SuperuserExtension {
+
   @XmlElement(name = "redemptionGracePeriodDays")
   int redemptionGracePeriodDays;
 

core/src/main/java/google/registry/model/domain/superuser/DomainTransferRequestSuperuserExtension.java

@@ -21,6 +21,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 /** A superuser extension that may be present on domain transfer request commands. */
 @XmlRootElement(name = "domainTransferRequest")
 public class DomainTransferRequestSuperuserExtension extends SuperuserExtension {
+
   // We need to specify the period here because the transfer object's period cannot be set to zero.
   @XmlElement(name = "renewalPeriod")
   Period renewalPeriod;

core/src/main/java/google/registry/model/domain/superuser/DomainUpdateSuperuserExtension.java

@@ -0,0 +1,35 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.domain.superuser;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import java.util.Optional;
+import javax.annotation.Nullable;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/** A superuser extension that may be present on domain update commands. */
+@XmlRootElement(name = "domainUpdate")
+public class DomainUpdateSuperuserExtension extends SuperuserExtension {
+
+  @XmlElement(name = "autorenews")
+  @Nullable
+  String autorenews;
+
+  public Optional<Boolean> getAutorenews() {
+    return Optional.ofNullable(isNullOrEmpty(autorenews) ? null : Boolean.valueOf(autorenews));
+  }
+}

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -49,16 +49,32 @@ import google.registry.model.common.TimedTransitionProperty.TimedTransition;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.Table;
 import org.joda.time.DateTime;
 
 /** An entity representing an allocation token. */
 @ReportedOn
 @Entity
 @WithStringVKey
-public class AllocationToken extends BackupGroupRoot implements Buildable {
+@javax.persistence.Entity
+@Table(
+    indexes = {
+      @javax.persistence.Index(
+          columnList = "token",
+          name = "allocation_token_token_idx",
+          unique = true),
+      @javax.persistence.Index(
+          columnList = "domainName",
+          name = "allocation_token_domain_name_idx"),
+    })
+public class AllocationToken extends BackupGroupRoot implements Buildable, DatastoreAndSqlEntity {
 
   // Promotions should only move forward, and ENDED / CANCELLED are terminal states.
   private static final ImmutableMultimap<TokenStatus, TokenStatus> VALID_TOKEN_STATUS_TRANSITIONS =
@@ -86,19 +102,22 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
   }
 
   /** The allocation token string. */
-  @Id String token;
+  @javax.persistence.Id @Id String token;
 
   /** The key of the history entry for which the token was used. Null if not yet used. */
-  @Nullable @Index Key<HistoryEntry> redemptionHistoryEntry;
+  @Nullable @Index VKey<HistoryEntry> redemptionHistoryEntry;
 
   /** The fully-qualified domain name that this token is limited to, if any. */
   @Nullable @Index String domainName;
 
   /** When this token was created. */
+  @Column(nullable = false)
   CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /** Allowed registrar client IDs for this token, or null if all registrars are allowed. */
-  @Nullable Set<String> allowedClientIds;
+  @Column(name = "allowedRegistrarIds")
+  @Nullable
+  Set<String> allowedClientIds;
 
   /** Allowed TLDs for this token, or null if all TLDs are allowed. */
   @Nullable Set<String> allowedTlds;
@@ -117,6 +136,7 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
   int discountYears = 1;
 
   /** The type of the token, either single-use or unlimited-use. */
+  @Enumerated(EnumType.STRING)
   TokenType tokenType;
 
   // TODO: Remove onLoad once all allocation tokens are migrated to have a discountYears of 1.
@@ -161,7 +181,7 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
     return token;
   }
 
-  public Optional<Key<HistoryEntry>> getRedemptionHistoryEntry() {
+  public Optional<VKey<HistoryEntry>> getRedemptionHistoryEntry() {
     return Optional.ofNullable(redemptionHistoryEntry);
   }
 
@@ -177,7 +197,7 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
     return Optional.ofNullable(creationTime.getTimestamp());
   }
 
-  public ImmutableSet<String> getAllowedClientIds() {
+  public ImmutableSet<String> getAllowedRegistrarIds() {
     return nullToEmptyImmutableCopy(allowedClientIds);
   }
 
@@ -260,7 +280,7 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
       return this;
     }
 
-    public Builder setRedemptionHistoryEntry(Key<HistoryEntry> redemptionHistoryEntry) {
+    public Builder setRedemptionHistoryEntry(VKey<HistoryEntry> redemptionHistoryEntry) {
       getInstance().redemptionHistoryEntry =
           checkArgumentNotNull(redemptionHistoryEntry, "Redemption history entry must not be null");
       return this;
@@ -279,8 +299,8 @@ public class AllocationToken extends BackupGroupRoot implements Buildable {
       return this;
     }
 
-    public Builder setAllowedClientIds(Set<String> allowedClientIds) {
-      getInstance().allowedClientIds = forceEmptyToNull(allowedClientIds);
+    public Builder setAllowedRegistrarIds(Set<String> allowedRegistrarIds) {
+      getInstance().allowedClientIds = forceEmptyToNull(allowedRegistrarIds);
       return this;
     }
 

core/src/main/java/google/registry/model/eppcommon/StatusValue.java

@@ -20,8 +20,10 @@ import static com.google.common.base.Strings.nullToEmpty;
 
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactBase;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
 import google.registry.model.host.HostBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.translators.EnumToAttributeAdapter.EppEnum;
@@ -128,9 +130,15 @@ public enum StatusValue implements EppEnum {
 
   /** Enum to help clearly list which resource types a status value is allowed to be present on. */
   private enum AllowedOn {
-    ALL(ContactResource.class, DomainBase.class, HostBase.class, HostResource.class),
+    ALL(
+        ContactBase.class,
+        ContactResource.class,
+        DomainContent.class,
+        DomainBase.class,
+        HostBase.class,
+        HostResource.class),
     NONE,
-    DOMAINS(DomainBase.class);
+    DOMAINS(DomainContent.class, DomainBase.class);
 
     private final ImmutableSet<Class<? extends EppResource>> classes;
 

core/src/main/java/google/registry/model/eppinput/EppInput.java

@@ -50,6 +50,7 @@ import google.registry.model.domain.secdns.SecDnsCreateExtension;
 import google.registry.model.domain.secdns.SecDnsUpdateExtension;
 import google.registry.model.domain.superuser.DomainDeleteSuperuserExtension;
 import google.registry.model.domain.superuser.DomainTransferRequestSuperuserExtension;
+import google.registry.model.domain.superuser.DomainUpdateSuperuserExtension;
 import google.registry.model.domain.token.AllocationTokenExtension;
 import google.registry.model.eppinput.ResourceCommand.ResourceCheck;
 import google.registry.model.eppinput.ResourceCommand.SingleResourceCommand;
@@ -309,53 +310,62 @@ public class EppInput extends ImmutableObject {
   @XmlType(propOrder = {"command", "extension", "clTRID"})
   public static class CommandWrapper extends ImmutableObject {
     @XmlElements({
-        @XmlElement(name = "check", type = Check.class),
-        @XmlElement(name = "create", type = Create.class),
-        @XmlElement(name = "delete", type = Delete.class),
-        @XmlElement(name = "info", type = Info.class),
-        @XmlElement(name = "login", type = Login.class),
-        @XmlElement(name = "logout", type = Logout.class),
-        @XmlElement(name = "poll", type = Poll.class),
-        @XmlElement(name = "renew", type = Renew.class),
-        @XmlElement(name = "transfer", type = Transfer.class),
-        @XmlElement(name = "update", type = Update.class) })
+      @XmlElement(name = "check", type = Check.class),
+      @XmlElement(name = "create", type = Create.class),
+      @XmlElement(name = "delete", type = Delete.class),
+      @XmlElement(name = "info", type = Info.class),
+      @XmlElement(name = "login", type = Login.class),
+      @XmlElement(name = "logout", type = Logout.class),
+      @XmlElement(name = "poll", type = Poll.class),
+      @XmlElement(name = "renew", type = Renew.class),
+      @XmlElement(name = "transfer", type = Transfer.class),
+      @XmlElement(name = "update", type = Update.class)
+    })
     InnerCommand command;
 
     /** Zero or more command extensions. */
     @XmlElementRefs({
-        // allocation token extension
-        @XmlElementRef(type = AllocationTokenExtension.class),
-        // fee extension version 0.6
-        @XmlElementRef(type = FeeCheckCommandExtensionV06.class),
-        @XmlElementRef(type = FeeInfoCommandExtensionV06.class),
-        @XmlElementRef(type = FeeCreateCommandExtensionV06.class),
-        @XmlElementRef(type = FeeRenewCommandExtensionV06.class),
-        @XmlElementRef(type = FeeTransferCommandExtensionV06.class),
-        @XmlElementRef(type = FeeUpdateCommandExtensionV06.class),
-        // fee extension version 0.11
-        @XmlElementRef(type = FeeCheckCommandExtensionV11.class),
-        @XmlElementRef(type = FeeCreateCommandExtensionV11.class),
-        @XmlElementRef(type = FeeRenewCommandExtensionV11.class),
-        @XmlElementRef(type = FeeTransferCommandExtensionV11.class),
-        @XmlElementRef(type = FeeUpdateCommandExtensionV11.class),
-        // fee extension version 0.12
-        @XmlElementRef(type = FeeCheckCommandExtensionV12.class),
-        @XmlElementRef(type = FeeCreateCommandExtensionV12.class),
-        @XmlElementRef(type = FeeRenewCommandExtensionV12.class),
-        @XmlElementRef(type = FeeTransferCommandExtensionV12.class),
-        @XmlElementRef(type = FeeUpdateCommandExtensionV12.class),
-        // other extensions
-        @XmlElementRef(type = LaunchCheckExtension.class),
-        @XmlElementRef(type = LaunchCreateExtension.class),
-        @XmlElementRef(type = LaunchDeleteExtension.class),
-        @XmlElementRef(type = LaunchInfoExtension.class),
-        @XmlElementRef(type = LaunchUpdateExtension.class),
-        @XmlElementRef(type = MetadataExtension.class),
-        @XmlElementRef(type = RgpUpdateExtension.class),
-        @XmlElementRef(type = SecDnsCreateExtension.class),
-        @XmlElementRef(type = SecDnsUpdateExtension.class),
-        @XmlElementRef(type = DomainTransferRequestSuperuserExtension.class),
-        @XmlElementRef(type = DomainDeleteSuperuserExtension.class) })
+      // Fee extension version 0.6
+      @XmlElementRef(type = FeeCheckCommandExtensionV06.class),
+      @XmlElementRef(type = FeeInfoCommandExtensionV06.class),
+      @XmlElementRef(type = FeeCreateCommandExtensionV06.class),
+      @XmlElementRef(type = FeeRenewCommandExtensionV06.class),
+      @XmlElementRef(type = FeeTransferCommandExtensionV06.class),
+      @XmlElementRef(type = FeeUpdateCommandExtensionV06.class),
+
+      // Fee extension version 0.11
+      @XmlElementRef(type = FeeCheckCommandExtensionV11.class),
+      @XmlElementRef(type = FeeCreateCommandExtensionV11.class),
+      @XmlElementRef(type = FeeRenewCommandExtensionV11.class),
+      @XmlElementRef(type = FeeTransferCommandExtensionV11.class),
+      @XmlElementRef(type = FeeUpdateCommandExtensionV11.class),
+
+      // Fee extension version 0.12
+      @XmlElementRef(type = FeeCheckCommandExtensionV12.class),
+      @XmlElementRef(type = FeeCreateCommandExtensionV12.class),
+      @XmlElementRef(type = FeeRenewCommandExtensionV12.class),
+      @XmlElementRef(type = FeeTransferCommandExtensionV12.class),
+      @XmlElementRef(type = FeeUpdateCommandExtensionV12.class),
+
+      // Launch phase extensions
+      @XmlElementRef(type = LaunchCheckExtension.class),
+      @XmlElementRef(type = LaunchCreateExtension.class),
+      @XmlElementRef(type = LaunchDeleteExtension.class),
+      @XmlElementRef(type = LaunchInfoExtension.class),
+      @XmlElementRef(type = LaunchUpdateExtension.class),
+
+      // Superuser extensions
+      @XmlElementRef(type = DomainDeleteSuperuserExtension.class),
+      @XmlElementRef(type = DomainTransferRequestSuperuserExtension.class),
+      @XmlElementRef(type = DomainUpdateSuperuserExtension.class),
+
+      // Other extensions
+      @XmlElementRef(type = AllocationTokenExtension.class),
+      @XmlElementRef(type = MetadataExtension.class),
+      @XmlElementRef(type = RgpUpdateExtension.class),
+      @XmlElementRef(type = SecDnsCreateExtension.class),
+      @XmlElementRef(type = SecDnsUpdateExtension.class)
+    })
     @XmlElementWrapper
     List<CommandExtension> extension;
 

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -14,12 +14,25 @@
 
 package google.registry.model.host;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
-import google.registry.model.EppResource;
+import com.googlecode.objectify.annotation.EntitySubclass;
+import google.registry.model.ImmutableObject;
+import google.registry.model.host.HostHistory.HostHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
 
 /**
  * A persisted history entry representing an EPP modification to a host.
@@ -37,22 +50,128 @@ import javax.persistence.Entity;
       @javax.persistence.Index(columnList = "historyType"),
       @javax.persistence.Index(columnList = "historyModificationTime")
     })
-public class HostHistory extends HistoryEntry {
+@EntitySubclass
+@Access(AccessType.FIELD)
+@IdClass(HostHistoryId.class)
+public class HostHistory extends HistoryEntry implements SqlEntity {
 
   // Store HostBase instead of HostResource so we don't pick up its @Id
-  HostBase hostBase;
+  @Nullable HostBase hostBase;
 
-  @Column(nullable = false)
-  VKey<HostResource> hostRepoId;
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getHostRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
 
-  /** The state of the {@link HostBase} object at this point in time. */
-  public HostBase getHostBase() {
-    return hostBase;
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setHostRepoId(String hostRepoId) {
+    parent = Key.create(HostResource.class, hostRepoId);
+  }
+
+  @Id
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
+  /**
+   * The values of all the fields on the {@link HostBase} object after the action represented by
+   * this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<HostBase> getHostBase() {
+    return Optional.ofNullable(hostBase);
   }
 
   /** The key to the {@link google.registry.model.host.HostResource} this is based off of. */
-  public VKey<HostResource> getHostRepoId() {
-    return hostRepoId;
+  public VKey<HostResource> getParentVKey() {
+    return VKey.create(HostResource.class, getHostRepoId());
+  }
+
+  /** Creates a {@link VKey} instance for this entity. */
+  public VKey<HostHistory> createVKey() {
+    return VKey.create(
+        HostHistory.class, new HostHistoryId(getHostRepoId(), getId()), Key.create(this));
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Normally Hibernate would see that the host fields are all null and would fill hostBase
+    // with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+    if (hostBase != null && hostBase.getHostName() == null) {
+      hostBase = null;
+    }
+  }
+
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
+  /** Class to represent the composite primary key of {@link HostHistory} entity. */
+  static class HostHistoryId extends ImmutableObject implements Serializable {
+
+    private String hostRepoId;
+
+    private Long id;
+
+    /** Hibernate requires this default constructor. */
+    private HostHistoryId() {}
+
+    HostHistoryId(String hostRepoId, long id) {
+      this.hostRepoId = hostRepoId;
+      this.id = id;
+    }
+
+    /**
+     * Returns the host repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getHostRepoId() {
+      return hostRepoId;
+    }
+
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
+      return id;
+    }
+
+    /**
+     * Sets the host repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setHostRepoId(String hostRepoId) {
+      this.hostRepoId = hostRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
+      this.id = id;
+    }
   }
 
   @Override
@@ -73,18 +192,8 @@ public class HostHistory extends HistoryEntry {
       return this;
     }
 
-    public Builder setHostRepoId(VKey<HostResource> hostRepoId) {
-      getInstance().hostRepoId = hostRepoId;
-      hostRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
-      return this;
-    }
-
-    // We can remove this once all HistoryEntries are converted to History objects
-    @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().hostRepoId =
-          VKey.create(HostResource.class, parent.getName(), (Key<HostResource>) parent);
+    public Builder setHostRepoId(String hostRepoId) {
+      getInstance().parent = Key.create(HostResource.class, hostRepoId);
       return this;
     }
   }

core/src/main/java/google/registry/model/host/HostResource.java

@@ -32,7 +32,7 @@ import javax.persistence.AccessType;
  */
 @ReportedOn
 @Entity
-@javax.persistence.Entity
+@javax.persistence.Entity(name = "Host")
 @ExternalMessagingName("host")
 @WithStringVKey
 @Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property)

core/src/main/java/google/registry/model/index/EppResourceIndex.java

@@ -17,6 +17,7 @@ package google.registry.model.index;
 import static google.registry.util.TypeUtils.instantiate;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -25,11 +26,13 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.EppResource;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /** An index that allows for quick enumeration of all EppResource entities (e.g. via map reduce). */
 @ReportedOn
 @Entity
-public class EppResourceIndex extends BackupGroupRoot {
+public class EppResourceIndex extends BackupGroupRoot implements DatastoreEntity {
 
   @Id
   String id;
@@ -74,4 +77,9 @@ public class EppResourceIndex extends BackupGroupRoot {
   public static <T extends EppResource> EppResourceIndex create(Key<T> resourceKey) {
     return create(EppResourceIndexBucket.getBucketKey(resourceKey), resourceKey);
   }
+
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not relevant in SQL
+  }
 }

core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java

@@ -24,16 +24,23 @@ import com.googlecode.objectify.annotation.Id;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.VirtualEntity;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /** A virtual entity to represent buckets to which EppResourceIndex objects are randomly added. */
 @Entity
 @VirtualEntity
-public class EppResourceIndexBucket extends ImmutableObject {
+public class EppResourceIndexBucket extends ImmutableObject implements DatastoreEntity {
 
   @SuppressWarnings("unused")
   @Id
   private long bucketId;
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not relevant in SQL
+  }
+
   /**
    * Deterministic function that returns a bucket id based on the resource's roid.
    * NB: At the moment, nothing depends on this being deterministic, so we have the ability to

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -20,7 +20,6 @@ import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntri
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.TypeUtils.instantiate;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
@@ -43,6 +42,8 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
 import java.util.Optional;
@@ -61,28 +62,44 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   /** The {@link ForeignKeyIndex} type for {@link ContactResource} entities. */
   @ReportedOn
   @Entity
-  public static class ForeignKeyContactIndex extends ForeignKeyIndex<ContactResource> {}
+  public static class ForeignKeyContactIndex extends ForeignKeyIndex<ContactResource>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }
 
   /** The {@link ForeignKeyIndex} type for {@link DomainBase} entities. */
   @ReportedOn
   @Entity
-  public static class ForeignKeyDomainIndex extends ForeignKeyIndex<DomainBase> {}
+  public static class ForeignKeyDomainIndex extends ForeignKeyIndex<DomainBase>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }
 
   /** The {@link ForeignKeyIndex} type for {@link HostResource} entities. */
   @ReportedOn
   @Entity
-  public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource> {}
+  public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }
 
-  static final ImmutableMap<
-          Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
+  static final ImmutableMap<Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
       RESOURCE_CLASS_TO_FKI_CLASS =
           ImmutableMap.of(
               ContactResource.class, ForeignKeyContactIndex.class,
               DomainBase.class, ForeignKeyDomainIndex.class,
               HostResource.class, ForeignKeyHostIndex.class);
 
-  @Id
-  String foreignKey;
+  @Id String foreignKey;
 
   /**
    * The deletion time of this {@link ForeignKeyIndex}.
@@ -90,8 +107,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * <p>This will generally be equal to the deletion time of {@link #topReference}. However, in the
    * case of a {@link HostResource} that was renamed, this field will hold the time of the rename.
    */
-  @Index
-  DateTime deletionTime;
+  @Index DateTime deletionTime;
 
   /**
    * The referenced resource.
@@ -227,7 +243,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   private static LoadingCache<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>>
       createForeignKeyIndexesCache(Duration expiry) {
     return CacheBuilder.newBuilder()
-        .expireAfterWrite(expiry.getMillis(), MILLISECONDS)
+        .expireAfterWrite(java.time.Duration.ofMillis(expiry.getMillis()))
         .maximumSize(getEppResourceMaxCachedEntries())
         .build(CACHE_LOADER);
   }

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -23,14 +23,19 @@ import com.google.common.base.Functions;
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
+import com.googlecode.objectify.Result;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.host.HostHistory;
+import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.TransactionManager;
-import java.util.Map.Entry;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.StreamSupport;
+import javax.annotation.Nullable;
 import org.joda.time.DateTime;
 
 /** Datastore implementation of {@link TransactionManager}. */
@@ -98,46 +103,73 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public void saveNew(Object entity) {
-    checkArgumentNotNull(entity, "entity must be specified");
-    getOfy().save().entity(entity);
+  public void insert(Object entity) {
+    put(entity);
   }
 
   @Override
-  public void saveAllNew(ImmutableCollection<?> entities) {
-    getOfy().save().entities(entities);
+  public void insertAll(ImmutableCollection<?> entities) {
+    putAll(entities);
   }
 
   @Override
-  public void saveNewOrUpdate(Object entity) {
-    checkArgumentNotNull(entity, "entity must be specified");
-    getOfy().save().entity(entity);
+  public void insertWithoutBackup(Object entity) {
+    putWithoutBackup(entity);
   }
 
   @Override
-  public void saveNewOrUpdateAll(ImmutableCollection<?> entities) {
-    getOfy().save().entities(entities);
+  public void insertAllWithoutBackup(ImmutableCollection<?> entities) {
+    putAllWithoutBackup(entities);
+  }
+
+  @Override
+  public void put(Object entity) {
+    saveEntity(entity);
+  }
+
+  @Override
+  public void putAll(ImmutableCollection<?> entities) {
+    syncIfTransactionless(getOfy().save().entities(entities));
+  }
+
+  @Override
+  public void putWithoutBackup(Object entity) {
+    syncIfTransactionless(getOfy().saveWithoutBackup().entities(entity));
+  }
+
+  @Override
+  public void putAllWithoutBackup(ImmutableCollection<?> entities) {
+    syncIfTransactionless(getOfy().saveWithoutBackup().entities(entities));
   }
 
   @Override
   public void update(Object entity) {
-    checkArgumentNotNull(entity, "entity must be specified");
-    getOfy().save().entity(entity);
+    put(entity);
   }
 
   @Override
   public void updateAll(ImmutableCollection<?> entities) {
-    getOfy().save().entities(entities);
+    putAll(entities);
   }
 
   @Override
-  public boolean checkExists(Object entity) {
+  public void updateWithoutBackup(Object entity) {
+    putWithoutBackup(entity);
+  }
+
+  @Override
+  public void updateAllWithoutBackup(ImmutableCollection<?> entities) {
+    putAllWithoutBackup(entities);
+  }
+
+  @Override
+  public boolean exists(Object entity) {
     return getOfy().load().key(Key.create(entity)).now() != null;
   }
 
   @Override
-  public <T> boolean checkExists(VKey<T> key) {
-    return getOfy().load().key(key.getOfyKey()).now() != null;
+  public <T> boolean exists(VKey<T> key) {
+    return loadNullable(key) != null;
   }
 
   // TODO: add tests for these methods.  They currently have some degree of test coverage because
@@ -146,18 +178,23 @@ public class DatastoreTransactionManager implements TransactionManager {
   // interface tests that are applied to both the datastore and SQL implementations.
   @Override
   public <T> Optional<T> maybeLoad(VKey<T> key) {
-    return Optional.ofNullable(getOfy().load().key(key.getOfyKey()).now());
+    return Optional.ofNullable(loadNullable(key));
   }
 
   @Override
   public <T> T load(VKey<T> key) {
-    T result = getOfy().load().key(key.getOfyKey()).now();
+    T result = loadNullable(key);
     if (result == null) {
       throw new NoSuchElementException(key.toString());
     }
     return result;
   }
 
+  @Override
+  public <T> T load(T entity) {
+    return ofy().load().entity(entity).now();
+  }
+
   @Override
   public <T> ImmutableMap<VKey<? extends T>, T> load(Iterable<? extends VKey<? extends T>> keys) {
     // Keep track of the Key -> VKey mapping so we can translate them back.
@@ -167,18 +204,25 @@ public class DatastoreTransactionManager implements TransactionManager {
             .collect(toImmutableMap(key -> (Key<T>) key.getOfyKey(), Functions.identity()));
 
     return getOfy().load().keys(keyMap.keySet()).entrySet().stream()
-        .collect(ImmutableMap.toImmutableMap(entry -> keyMap.get(entry.getKey()), Entry::getValue));
+        .collect(
+            toImmutableMap(
+                entry -> keyMap.get(entry.getKey()),
+                entry -> toChildHistoryEntryIfPossible(entry.getValue())));
   }
 
   @Override
   public <T> ImmutableList<T> loadAll(Class<T> clazz) {
-    // We can do a ofy().load().type(clazz), but this doesn't work in a transaction.
-    throw new UnsupportedOperationException("Not available in the Datastore transaction manager");
+    return ImmutableList.copyOf(getOfy().load().type(clazz));
+  }
+
+  @Override
+  public <T> ImmutableList<T> loadAll(Iterable<T> entities) {
+    return ImmutableList.copyOf(getOfy().load().entities(entities).values());
   }
 
   @Override
   public void delete(VKey<?> key) {
-    getOfy().delete().key(key.getOfyKey()).now();
+    syncIfTransactionless(getOfy().delete().key(key.getOfyKey()));
   }
 
   @Override
@@ -189,6 +233,82 @@ public class DatastoreTransactionManager implements TransactionManager {
         StreamSupport.stream(vKeys.spliterator(), false)
             .map(VKey::getOfyKey)
             .collect(toImmutableList());
-    getOfy().delete().keys(list).now();
+    syncIfTransactionless(getOfy().delete().keys(list));
+  }
+
+  @Override
+  public void delete(Object entity) {
+    syncIfTransactionless(getOfy().delete().entity(entity));
+  }
+
+  @Override
+  public void deleteWithoutBackup(VKey<?> key) {
+    syncIfTransactionless(getOfy().deleteWithoutBackup().key(key.getOfyKey()));
+  }
+
+  @Override
+  public void deleteWithoutBackup(Iterable<? extends VKey<?>> keys) {
+    syncIfTransactionless(
+        getOfy()
+            .deleteWithoutBackup()
+            .keys(Streams.stream(keys).map(VKey::getOfyKey).collect(toImmutableList())));
+  }
+
+  @Override
+  public void deleteWithoutBackup(Object entity) {
+    syncIfTransactionless(getOfy().deleteWithoutBackup().entity(entity));
+  }
+
+  @Override
+  public void clearSessionCache() {
+    getOfy().clearSessionCache();
+  }
+
+  /**
+   * The following three methods exist due to the migration to Cloud SQL.
+   *
+   * <p>In Cloud SQL, {@link HistoryEntry} objects are represented instead as {@link DomainHistory},
+   * {@link ContactHistory}, and {@link HostHistory} objects. During the migration, we do not wish
+   * to change the Datastore schema so all of these objects are stored in Datastore as HistoryEntry
+   * objects. They are converted to/from the appropriate classes upon retrieval, and converted to
+   * HistoryEntry on save. See go/r3.0-history-objects for more details.
+   */
+  private void saveEntity(Object entity) {
+    checkArgumentNotNull(entity, "entity must be specified");
+    if (entity instanceof HistoryEntry) {
+      entity = ((HistoryEntry) entity).asHistoryEntry();
+    }
+    syncIfTransactionless(getOfy().save().entity(entity));
+  }
+
+  @SuppressWarnings("unchecked")
+  private <T> T toChildHistoryEntryIfPossible(@Nullable T obj) {
+    // NB: The Key of the object in question may not necessarily be the resulting class that we
+    // wish to have. Because all *History classes are @EntitySubclasses, their Keys will have type
+    // HistoryEntry -- even if you create them based off the *History class.
+    if (obj != null && HistoryEntry.class.isAssignableFrom(obj.getClass())) {
+      return (T) ((HistoryEntry) obj).toChildHistoryEntity();
+    }
+    return obj;
+  }
+
+  @Nullable
+  private <T> T loadNullable(VKey<T> key) {
+    return toChildHistoryEntryIfPossible(getOfy().load().key(key.getOfyKey()).now());
+  }
+
+  /**
+   * Executes the given {@link Result} instance synchronously if not in a transaction.
+   *
+   * <p>The {@link Result} instance contains a task that will be executed by Objectify
+   * asynchronously. If it is in a transaction, we don't need to execute the task immediately
+   * because it is guaranteed to be done by the end of the transaction. However, if it is not in a
+   * transaction, we need to execute it in case the following code expects that happens before
+   * themselves.
+   */
+  private void syncIfTransactionless(Result<?> result) {
+    if (!inTransaction()) {
+      result.now();
+    }
   }
 }