Add a command to generate ER diagram for SQL schema (#839)

* Add a command to generate ER diagram for SQL schema

* Add graphviz as runtime dependency

* Update ER diagrams for #838

build.gradle

@@ -191,7 +191,7 @@ allprojects {
 }
 
 task runPresubmits(type: Exec) {
-  executable '/usr/bin/python'
+  executable '/usr/bin/python3'
   args('config/presubmits.py')
 }
 
@@ -260,7 +260,7 @@ subprojects {
       // in the 'configurations' block, the following code must run after
       // project evaluation, when all configurations have been created.
       configurations.each {
-        if (it.name != 'dependencyLicenseReport') {
+        if (it.name != 'dependencyLicenseReport' && it.name != 'integration') {
           it.resolutionStrategy.activateDependencyLocking()
         }
       }

config/nom_build.py

@@ -16,7 +16,7 @@
 """
 
 import argparse
-import attr
+import dataclasses
 import io
 import os
 import shutil
@@ -25,7 +25,7 @@ import sys
 from typing import List, Union
 
 
-@attr.s(auto_attribs=True)
+@dataclasses.dataclass
 class Property:
     name : str = ''
     desc : str = ''
@@ -39,7 +39,7 @@ class Property:
                 raise ValidationError('value of {self.name} must be "true" or '
                                       '"false".')
 
-@attr.s(auto_attribs=True)
+@dataclasses.dataclass
 class GradleFlag:
     flags : Union[str, List[str]]
     desc : str
@@ -75,6 +75,7 @@ Pseudo-commands:
 """
 
 # Define all of our special gradle properties here.
+# TODO(b/169318491): use consistent naming style for properties and variables.
 PROPERTIES = [
     Property('mavenUrl',
              'URL to use for the main maven repository (defaults to maven '
@@ -124,6 +125,9 @@ PROPERTIES = [
              'server/schema integration tests. Please refer to <a '
              'href="./integration/README.md">integration project</a> for more '
              'information.'),
+    Property('baseSchemaTag',
+             'The nomulus version tag of the schema for use in the schema'
+             'deployment integration test (:db:schemaIncrementalDeployTest)'),
     Property('schema_version',
              'The nomulus version tag of the schema for use in a database'
              'integration test.'),

config/presubmits.py

@@ -18,6 +18,7 @@ Error Prone) so we must write them manually.
 """
 
 import os
+from typing import List, Tuple
 import sys
 import re
 
@@ -178,6 +179,90 @@ PRESUBMITS = {
         "JavaScript files should not include console logging."
 }
 
+# Note that this regex only works for one kind of Flyway file.  If we want to
+# start using "R" and "U" files we'll need to update this script.
+FLYWAY_FILE_RX = re.compile(r'V(\d+)__.*')
+
+
+def get_seqnum(filename: str, location: str) -> int:
+  """Extracts the sequence number from a filename."""
+  m = FLYWAY_FILE_RX.match(filename)
+  if m is None:
+    raise ValueError('Illegal Flyway filename: %s in %s' % (filename, location))
+  return int(m.group(1))
+
+
+def files_by_seqnum(files: List[str], location: str) -> List[Tuple[int, str]]:
+  """Returns the list of seqnum, filename sorted by sequence number."""
+  return [(get_seqnum(filename, location), filename) for filename in files]
+
+
+def has_valid_order(indexed_files: List[Tuple[int, str]], location: str) -> bool:
+  """Verify that sequence numbers are in order without gaps or duplicates.
+
+  Args:
+    files: List of seqnum, filename for a list of Flyway files.
+    location: Where the list of files came from (for error reporting).
+
+  Returns:
+    True if the file list is valid.
+  """
+  last_index = 0
+  valid = True
+  for seqnum, filename in indexed_files:
+    if seqnum == last_index:
+      print('duplicate Flyway file sequence number found in %s: %s' %
+            (location, filename))
+      valid = False
+    elif seqnum < last_index:
+      print('File %s in %s is out of order.' % (filename, location))
+      valid = False
+    elif seqnum != last_index + 1:
+      print('Missing Flyway sequence number %d in %s.  Next file is %s' %
+            (last_index + 1, location, filename))
+      valid = False
+    last_index = seqnum
+  return valid
+
+
+def verify_flyway_index():
+  """Verifies that the Flyway index file is in sync with the directory."""
+  success = True
+
+  # Sort the files in the Flyway directory by their sequence number.
+  files = sorted(
+      files_by_seqnum(os.listdir('db/src/main/resources/sql/flyway'),
+                      'Flyway directory'))
+
+  # Make sure that there are no gaps and no duplicate sequence numbers in the
+  # files themselves.
+  if not has_valid_order(files, 'Flyway directory'):
+    success = False
+
+  # Remove the sequence numbers and compare against the index file contents.
+  files = [filename[1] for filename in sorted(files)]
+  with open('db/src/main/resources/sql/flyway.txt') as index:
+    indexed_files = index.read().splitlines()
+  if files != indexed_files:
+    unindexed = set(files) - set(indexed_files)
+    if unindexed:
+      print('The following Flyway files are not in flyway.txt: %s' % unindexed)
+
+    nonexistent = set(indexed_files) - set(files)
+    if nonexistent:
+      print('The following files are in flyway.txt but not in the Flyway '
+            'directory: %s' % nonexistent)
+
+    # Do an ordering check on the index file (ignore the result, we're failing
+    # anyway).
+    has_valid_order(files_by_seqnum(indexed_files, 'flyway.txt'), 'flyway.txt')
+    success = False
+
+  if not success:
+    print('Please fix any conflicts and run "./nom_build :db:generateFlywayIndex"')
+
+  return not success
+
 
 def get_files():
   for root, dirnames, filenames in os.walk("."):
@@ -197,5 +282,10 @@ if __name__ == "__main__":
       failed = True
       print("%s had errors: \n  %s" % (file, "\n  ".join(error_messages)))
 
+  # And now for something completely different: check to see if the Flyway
+  # index is up-to-date.  It's quicker to do it here than in the unit tests:
+  # when we put it here it fails fast before all of the tests are run.
+  failed |= verify_flyway_index()
+
   if failed:
     sys.exit(1)

core/build.gradle

@@ -221,6 +221,7 @@ dependencies {
   compile deps['com.jcraft:jsch']
   testCompile deps['com.thoughtworks.qdox:qdox']
   compile deps['dnsjava:dnsjava']
+  runtime deps['guru.nidi:graphviz-java-all-j2v8']
   testCompile deps['io.github.classgraph:classgraph']
   testRuntime deps['io.github.java-diff-utils:java-diff-utils']
   testCompile deps['javax.annotation:javax.annotation-api']
@@ -245,6 +246,7 @@ dependencies {
   testCompile deps['org.apache.ftpserver:ftpserver-core']
   compile deps['org.apache.httpcomponents:httpclient']
   compile deps['org.apache.httpcomponents:httpcore']
+  runtime deps['org.apache.logging.log4j:log4j-core']
   testCompile deps['org.apache.sshd:sshd-core']
   testCompile deps['org.apache.sshd:sshd-scp']
   testCompile deps['org.apache.sshd:sshd-sftp']
@@ -257,6 +259,7 @@ dependencies {
   compile deps['org.hibernate:hibernate-core']
   compile deps['org.joda:joda-money']
   compile deps['org.json:json']
+  compile deps['org.jsoup:jsoup']
   testCompile deps['org.mortbay.jetty:jetty']
   compile deps['org.postgresql:postgresql']
   testCompile deps['org.seleniumhq.selenium:selenium-api']
@@ -269,6 +272,10 @@ dependencies {
   compile deps['org.testcontainers:postgresql']
   testCompile deps['org.testcontainers:selenium']
   testCompile deps['org.testcontainers:testcontainers']
+  compile deps['us.fatehi:schemacrawler']
+  compile deps['us.fatehi:schemacrawler-api']
+  compile deps['us.fatehi:schemacrawler-diagram']
+  compile deps['us.fatehi:schemacrawler-tools']
   compile deps['xerces:xmlParserAPIs']
   compile deps['xpp3:xpp3']
   // This dependency must come after javax.mail:mail as it would otherwise

core/gradle/dependency-locks/compile.lockfile

@@ -202,6 +202,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +226,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -249,5 +251,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -200,6 +200,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -222,6 +223,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -246,5 +248,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/default.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -177,6 +184,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -198,11 +206,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -226,6 +237,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -240,6 +252,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -249,7 +263,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +235,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -238,6 +250,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -202,6 +202,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +226,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -249,5 +251,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -201,6 +201,7 @@ org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
 org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +225,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -248,5 +250,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/runtime.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -129,6 +133,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.12
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -225,6 +236,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -239,6 +251,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.2
 com.fasterxml.jackson.core:jackson-core:2.10.2
 com.fasterxml.jackson.core:jackson-databind:2.10.2
@@ -130,6 +134,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.grpc:grpc-all:1.27.2
@@ -176,6 +183,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
 net.java.dev.jna:jna:5.5.0
@@ -197,11 +205,14 @@ org.apache.beam:beam-vendor-grpc-1_26_0:0.3
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
 org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.23.0
 org.apache.commons:commons-compress:1.20
+org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.5
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
+org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
 org.checkerframework:checker-qual:2.11.1
@@ -224,6 +235,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.jvnet.staxex:stax-ex:1.8
 org.mortbay.jetty:jetty-util:6.1.26
 org.mortbay.jetty:jetty:6.1.26
@@ -238,6 +250,8 @@ org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.rnorth:tcp-unix-socket-proxy:1.0.2
 org.scijava:native-lib-loader:2.0.2
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -247,7 +261,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testCompile.lockfile

@@ -248,6 +248,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -298,5 +299,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -247,6 +247,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -297,5 +298,10 @@ org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testRuntime.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.11.2
 com.fasterxml.jackson.core:jackson-core:2.11.2
 com.fasterxml.jackson.core:jackson-databind:2.11.2
@@ -138,6 +142,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.github.java-diff-utils:java-diff-utils:4.0
@@ -186,6 +193,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.13
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
@@ -216,7 +224,8 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -251,6 +260,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -289,6 +299,8 @@ org.seleniumhq.selenium:selenium-opera-driver:3.141.59
 org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.testcontainers:database-commons:1.14.3
 org.testcontainers:jdbc:1.14.3
@@ -299,7 +311,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -6,6 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
+com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
+com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
 com.fasterxml.jackson.core:jackson-annotations:2.11.2
 com.fasterxml.jackson.core:jackson-core:2.11.2
 com.fasterxml.jackson.core:jackson-databind:2.11.2
@@ -138,6 +142,9 @@ com.zaxxer:HikariCP:3.2.0
 commons-codec:commons-codec:1.13
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:2.1.7
+guru.nidi.com.kitfox:svgSalamander:1.1.3
+guru.nidi:graphviz-java-all-j2v8:0.17.0
+guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
 io.github.java-diff-utils:java-diff-utils:4.0
@@ -186,6 +193,7 @@ javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
 junit:junit:4.13
+net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.5
 net.bytebuddy:byte-buddy:1.10.10
 net.java.dev.jna:jna-platform:5.5.0
@@ -216,7 +224,8 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.11
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.logging.log4j:log4j-api:2.6.2
+org.apache.logging.log4j:log4j-api:2.13.3
+org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -251,6 +260,7 @@ org.jboss:jandex:2.1.3.Final
 org.jetbrains:annotations:19.0.0
 org.joda:joda-money:1.0.1
 org.json:json:20160810
+org.jsoup:jsoup:1.13.1
 org.junit-pioneer:junit-pioneer:0.7.0
 org.junit.jupiter:junit-jupiter-api:5.6.2
 org.junit.jupiter:junit-jupiter-engine:5.6.2
@@ -289,6 +299,8 @@ org.seleniumhq.selenium:selenium-opera-driver:3.141.59
 org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
+org.slf4j:jcl-over-slf4j:1.7.30
+org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
 org.testcontainers:database-commons:1.14.3
@@ -300,7 +312,13 @@ org.testcontainers:testcontainers:1.14.3
 org.threeten:threetenbp:1.4.1
 org.tukaani:xz:1.8
 org.w3c.css:sac:1.3
+org.webjars.npm:viz.js-for-graphviz-java:2.1.3
 org.xerial.snappy:snappy-java:1.1.4
 org.yaml:snakeyaml:1.17
+us.fatehi:schemacrawler-api:16.10.1
+us.fatehi:schemacrawler-diagram:16.10.1
+us.fatehi:schemacrawler-tools:16.10.1
+us.fatehi:schemacrawler-utility:16.10.1
+us.fatehi:schemacrawler:16.10.1
 xerces:xmlParserAPIs:2.6.2
 xpp3:xpp3:1.1.4c

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -464,7 +464,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
         HostResource host = (HostResource) existingResource;
         if (host.isSubordinate()) {
           dnsQueue.addHostRefreshTask(host.getHostName());
-          tm().saveNewOrUpdate(
+          tm().put(
                   tm().load(host.getSuperordinateDomain())
                       .asBuilder()
                       .removeSubordinateHost(host.getHostName())

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -437,7 +437,7 @@ public final class Transforms {
                 .map(Optional::get)
                 .map(ofy::toPojo)
                 .collect(ImmutableList.toImmutableList());
-        retry(() -> jpaTm().transact(() -> jpaTm().saveNewOrUpdateAll(ofyEntities)));
+        retry(() -> jpaTm().transact(() -> jpaTm().putAll(ofyEntities)));
       }
     }
 

core/src/main/java/google/registry/beam/invoicing/sql/billing_events.sql

@@ -55,9 +55,7 @@ FROM (
     FROM
       `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
     WHERE
-    -- TODO(b/18092292): Add a filter for tldState (not PDT/PREDELEGATION)
-      tldType = 'REAL'
-    AND disableInvoicing is not TRUE) ) AS BillingEvent
+      invoicingEnabled IS TRUE) ) AS BillingEvent
   -- Gather billing ID from registrar table
   -- This is a 'JOIN' as opposed to 'LEFT JOIN' to filter out
   -- non-billable registrars

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -215,7 +215,7 @@ public class Spec11Pipeline implements Serializable {
                           .setRegistrarId(subdomain.registrarId())
                           .build();
                   JpaTransactionManager jpaTransactionManager = jpaSupplierFactory.get();
-                  jpaTransactionManager.transact(() -> jpaTransactionManager.saveNew(threatMatch));
+                  jpaTransactionManager.transact(() -> jpaTransactionManager.insert(threatMatch));
                 }
               }
             }));

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -173,7 +173,7 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
             existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId);
     updateForeignKeyIndexDeletionTime(newDomain);
     entitiesToSave.add(newDomain, historyEntry, autorenewEvent, autorenewPollMessage);
-    tm().saveNewOrUpdateAll(entitiesToSave.build());
+    tm().putAll(entitiesToSave.build());
     tm().delete(existingDomain.getDeletePollMessage());
     dnsQueue.addDomainRefreshTask(existingDomain.getDomainName());
     return responseBuilder

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -67,6 +67,7 @@ import google.registry.model.domain.DomainCommand.Update.AddRemove;
 import google.registry.model.domain.DomainCommand.Update.Change;
 import google.registry.model.domain.fee.FeeUpdateCommandExtension;
 import google.registry.model.domain.metadata.MetadataExtension;
+import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.domain.secdns.SecDnsUpdateExtension;
 import google.registry.model.domain.superuser.DomainUpdateSuperuserExtension;
 import google.registry.model.eppcommon.AuthInfo;
@@ -238,10 +239,16 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     DomainBase.Builder domainBuilder =
         domain
             .asBuilder()
-            // Handle the secDNS extension.
+            // Handle the secDNS extension. As dsData in secDnsUpdate is read from EPP input and
+            // does not have domainRepoId set, we create a copy of the existing dsData without
+            // domainRepoId for comparison.
             .setDsData(
                 secDnsUpdate.isPresent()
-                    ? updateDsData(domain.getDsData(), secDnsUpdate.get())
+                    ? updateDsData(
+                        domain.getDsData().stream()
+                            .map(DelegationSignerData::cloneWithoutDomainRepoId)
+                            .collect(toImmutableSet()),
+                        secDnsUpdate.get())
                     : domain.getDsData())
             .setLastEppUpdateTime(now)
             .setLastEppUpdateClientId(clientId)

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -285,7 +285,7 @@ public final class HostUpdateFlow implements TransactionalFlow {
         && newHost.isSubordinate()
         && Objects.equals(
             existingHost.getSuperordinateDomain(), newHost.getSuperordinateDomain())) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(existingHost.getSuperordinateDomain())
                   .asBuilder()
                   .removeSubordinateHost(existingHost.getHostName())
@@ -294,14 +294,14 @@ public final class HostUpdateFlow implements TransactionalFlow {
       return;
     }
     if (existingHost.isSubordinate()) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(existingHost.getSuperordinateDomain())
                   .asBuilder()
                   .removeSubordinateHost(existingHost.getHostName())
                   .build());
     }
     if (newHost.isSubordinate()) {
-      tm().saveNewOrUpdate(
+      tm().put(
               tm().load(newHost.getSuperordinateDomain())
                   .asBuilder()
                   .addSubordinateHost(newHost.getHostName())

core/src/main/java/google/registry/flows/poll/PollFlowUtils.java

@@ -64,7 +64,7 @@ public final class PollFlowUtils {
       // and re-save it for future autorenew poll messages to be delivered. Otherwise, this
       // autorenew poll message has no more events to deliver and should be deleted.
       if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
-        tm().saveNewOrUpdate(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
+        tm().put(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
         includeAckedMessageInCount = isBeforeOrAt(nextEventTime, tm().getTransactionTime());
       } else {
         tm().delete(autorenewPollMessage.createVKey());

core/src/main/java/google/registry/idn/Arabic-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Arab
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Armenian-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Armn
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Bengali-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Beng
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Chinese-IDN.txt

@@ -1,24 +1,24 @@
 # Registry: Charleston Road Registry Inc.
-# Script: Chinese
+# Script: zh-Hans
 # Version: 1.0
 # Effective Date: 04-12-2012
 #
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com
 # Notes: This table describes codepoints allowed for the Chinese script.
 
-Reference 0 Unicode 3.2
-Reference 1 A Complete Set of Simplified Chinese Characters
-Reference 2 Chinese Variants Collation Table
-Reference 3 Chinese Big Dictionary
-Reference 4 Chinese Relationship Table for Unihan Project
-Reference 5 GB2312
-Reference 6 General Table for Modern Chinese
-Reference 7 International Chinese Standard Big Dictionary
-Reference 8 Unihan Database
-Reference 9 BIG5
+# Reference 0 Unicode 3.2
+# Reference 1 A Complete Set of Simplified Chinese Characters
+# Reference 2 Chinese Variants Collation Table
+# Reference 3 Chinese Big Dictionary
+# Reference 4 Chinese Relationship Table for Unihan Project
+# Reference 5 GB2312
+# Reference 6 General Table for Modern Chinese
+# Reference 7 International Chinese Standard Big Dictionary
+# Reference 8 Unihan Database
+# Reference 9 BIG5
 
 U+002D(0);U+002D(0);
 U+0030(0);U+0030(0);

core/src/main/java/google/registry/idn/Cyrillic-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Cyrl
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Devanagari-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Deva
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Ethiopic-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Ethi
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Georgian-IDN.txt

@@ -3,7 +3,7 @@
 #
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Greek-IDN.txt

@@ -3,7 +3,7 @@
 # Script: Grek
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Gurmukhi-IDN.txt

@@ -3,7 +3,7 @@
 # Script: Guru
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Hebrew-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Hebr
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Kannada-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Knda
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Khmer-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Khmr
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 #
 # Telephone: +1 (650) 253-0000

core/src/main/java/google/registry/idn/Korean-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Kore
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Malayalam-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Mlym
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Myanmar-IDN.txt

@@ -3,7 +3,7 @@
 # Version: 1.0
 # Effective Date: 04-12-2012
 #
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Oriya-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Orya
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Sinhala-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Sinh
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Tamil-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Taml
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Telugu-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Telu
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Thai-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Thai
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Tibetan-IDN.txt

@@ -2,7 +2,7 @@
 # Script: Tibt
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com

core/src/main/java/google/registry/idn/Traditional-Chinese-IDN.txt

@@ -1,23 +1,23 @@
 # Registry: Charleston Road Registry Inc.
-# Script: Traditional Chinese
+# Script: zh-Hant
 # Version: 1.0
 # Effective Date: 04-12-2012
-# Contact: tas-contact.google.com
+# Contact: iana-contact@google.com
 # Address: 1600 Amphitheatre Parkway Mountain View, CA 94043, USA
 # Telephone: +1 (650) 253-0000
 # Website: www.google.com
 # Notes: This table describes codepoints allowed for the Traditional Chinese script.
 
-Reference 0 Unicode 3.2
-Reference 1 A Complete Set of Simplified Chinese Characters
-Reference 2 Chinese Variants Collation Table
-Reference 3 Chinese Big Dictionary
-Reference 4 Chinese Relationship Table for Unihan Project
-Reference 5 GB2312
-Reference 6 General Table for Modern Chinese
-Reference 7 International Chinese Standard Big Dictionary
-Reference 8 Unihan Database
-Reference 9 BIG5
+# Reference 0 Unicode 3.2
+# Reference 1 A Complete Set of Simplified Chinese Characters
+# Reference 2 Chinese Variants Collation Table
+# Reference 3 Chinese Big Dictionary
+# Reference 4 Chinese Relationship Table for Unihan Project
+# Reference 5 GB2312
+# Reference 6 General Table for Modern Chinese
+# Reference 7 International Chinese Standard Big Dictionary
+# Reference 8 Unihan Database
+# Reference 9 BIG5
 
 U+002D(0);U+002D(0);
 U+0030(0);U+0030(0);

core/src/main/java/google/registry/model/EppResource.java

@@ -62,23 +62,32 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   /**
    * Unique identifier in the registry for this resource.
    *
+   * <p>Not persisted so that we can store these in references to other objects. Subclasses that
+   * wish to use this as the primary key should create a getter method annotated with @Id
+   *
    * <p>This is in the (\w|_){1,80}-\w{1,8} format specified by RFC 5730 for roidType.
    *
    * @see <a href="https://tools.ietf.org/html/rfc5730">RFC 5730</a>
    */
-  @Id
-  // not persisted so that we can store these in references to other objects. Subclasses that wish
-  // to use this as the primary key should create a getter method annotated with @Id
-  @Transient
-  String repoId;
+  @Id @Transient String repoId;
 
-  /** The ID of the registrar that is currently sponsoring this resource. */
+  /**
+   * The ID of the registrar that is currently sponsoring this resource.
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
   @Index
-  @Column(name = "currentSponsorRegistrarId", nullable = false)
+  @Column(name = "currentSponsorRegistrarId")
   String currentSponsorClientId;
 
-  /** The ID of the registrar that created this resource. */
-  @Column(name = "creationRegistrarId", nullable = false)
+  /**
+   * The ID of the registrar that created this resource.
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
+  @Column(name = "creationRegistrarId")
   String creationClientId;
 
   /**
@@ -91,13 +100,17 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   @Column(name = "lastEppUpdateRegistrarId")
   String lastEppUpdateClientId;
 
-  /** The time when this resource was created. */
-  // Map the method to XML, not the field, because if we map the field (with an adaptor class) it
-  // will never be omitted from the xml even if the timestamp inside creationTime is null and we
-  // return null from the adaptor. (Instead it gets written as an empty tag.)
-  @Column(nullable = false)
-  @Index
-  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
+  /**
+   * The time when this resource was created.
+   *
+   * <p>Map the method to XML, not the field, because if we map the field (with an adaptor class) it
+   * will never be omitted from the xml even if the timestamp inside creationTime is null and we
+   * return null from the adaptor (instead it gets written as an empty tag).
+   *
+   * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
+   * resource fields.
+   */
+  @Index CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.
@@ -112,8 +125,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * out of the index at that time, as long as we query for resources whose deletion time is before
    * now.
    */
-  @Index
-  DateTime deletionTime;
+  @Index DateTime deletionTime;
 
   /**
    * The time that this resource was last updated.
@@ -144,7 +156,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
     return repoId;
   }
 
-  // Hibernate needs this to populate the repo ID, but no one else should ever use it
+  /** This method exists solely to satisfy Hibernate. Use {@link Builder} instead. */
   @SuppressWarnings("UnusedMethod")
   private void setRepoId(String repoId) {
     this.repoId = repoId;

core/src/main/java/google/registry/model/OteAccountBuilder.java

@@ -252,7 +252,8 @@ public final class OteAccountBuilder {
   private void saveAllEntities() {
     tm().assertInTransaction();
 
-    ImmutableList<Registry> registries = ImmutableList.of(sunriseTld, gaTld, eapTld);
+    // use ImmutableObject instead of Registry so that the Key generation doesn't break
+    ImmutableList<ImmutableObject> registries = ImmutableList.of(sunriseTld, gaTld, eapTld);
     ImmutableList<RegistrarContact> contacts = contactsBuilder.build();
 
     if (!replaceExisting) {

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -33,12 +33,14 @@ import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.TimeOfYear;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.GracePeriod;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.token.AllocationToken;
@@ -46,6 +48,7 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -56,16 +59,14 @@ import javax.persistence.Column;
 import javax.persistence.Embedded;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.MappedSuperclass;
+import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
 /** A billable event in a domain's lifecycle. */
 @MappedSuperclass
-@WithLongVKey
 public abstract class BillingEvent extends ImmutableObject
     implements Buildable, TransferServerApproveEntity {
 
@@ -107,10 +108,7 @@ public abstract class BillingEvent extends ImmutableObject
   }
 
   /** Entity id. */
-  @Id
-  @javax.persistence.Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
-  Long id;
+  @Id @javax.persistence.Id Long id;
 
   @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
 
@@ -148,6 +146,21 @@ public abstract class BillingEvent extends ImmutableObject
   @Nullable
   Set<Flag> flags;
 
+  @PostLoad
+  void postLoad() {
+    parent =
+        Key.create(
+            Key.create(DomainBase.class, domainRepoId),
+            HistoryEntry.class,
+            domainHistoryRevisionId);
+  }
+
+  @OnLoad
+  void onLoad() {
+    domainHistoryRevisionId = parent.getId();
+    domainRepoId = parent.getParent().getName();
+  }
+
   public String getClientId() {
     return clientId;
   }
@@ -244,7 +257,6 @@ public abstract class BillingEvent extends ImmutableObject
     }
 
     public B setParent(Key<HistoryEntry> parentKey) {
-      // TODO(shicong): Figure out how to set domainHistoryRevisionId and domainRepoId
       getInstance().parent = parentKey;
       return thisCastToDerived();
     }
@@ -257,6 +269,11 @@ public abstract class BillingEvent extends ImmutableObject
       checkNotNull(instance.eventTime, "Event time must be set");
       checkNotNull(instance.targetId, "Target ID must be set");
       checkNotNull(instance.parent, "Parent must be set");
+      checkNotNull(instance.parent.getParent(), "parent.getParent() must be set");
+      checkNotNull(
+          instance.parent.getParent().getName(), "parent.getParent().getName() must be set");
+      instance.domainHistoryRevisionId = instance.parent.getId();
+      instance.domainRepoId = instance.parent.getParent().getName();
       return super.build();
     }
   }
@@ -274,7 +291,8 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "allocation_token_id")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
-  public static class OneTime extends BillingEvent {
+  @WithLongVKey
+  public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
     @AttributeOverrides({
@@ -309,7 +327,7 @@ public abstract class BillingEvent extends ImmutableObject
      * Cancellation}s.
      */
     @Column(name = "cancellation_matching_billing_recurrence_id")
-    VKey<? extends BillingEvent> cancellationMatchingBillingEvent;
+    VKey<Recurring> cancellationMatchingBillingEvent;
 
     /**
      * The {@link AllocationToken} used in the creation of this event, or null if one was not used.
@@ -391,7 +409,7 @@ public abstract class BillingEvent extends ImmutableObject
       }
 
       public Builder setCancellationMatchingBillingEvent(
-          VKey<? extends BillingEvent> cancellationMatchingBillingEvent) {
+          VKey<Recurring> cancellationMatchingBillingEvent) {
         getInstance().cancellationMatchingBillingEvent = cancellationMatchingBillingEvent;
         return this;
       }
@@ -450,7 +468,8 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
-  public static class Recurring extends BillingEvent {
+  @WithLongVKey
+  public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity {
 
     /**
      * The billing event recurs every year between {@link #eventTime} and this time on the
@@ -544,7 +563,8 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "billingTime")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
-  public static class Cancellation extends BillingEvent {
+  @WithLongVKey
+  public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billing time of the charge that is being cancelled. */
     @Index
@@ -664,7 +684,8 @@ public abstract class BillingEvent extends ImmutableObject
   /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
-  public static class Modification extends BillingEvent {
+  @WithLongVKey
+  public static class Modification extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The change in cost that should be applied to the original billing event. */
     Money cost;

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -14,18 +14,25 @@
 
 package google.registry.model.contact;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
-import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.contact.ContactHistory.ContactHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
 
 /**
  * A persisted history entry representing an EPP modification to a contact.
@@ -44,15 +51,25 @@ import javax.persistence.Id;
     })
 @EntitySubclass
 @Access(AccessType.FIELD)
-public class ContactHistory extends HistoryEntry {
-  // Store ContactBase instead of ContactResource so we don't pick up its @Id
-  ContactBase contactBase;
+@IdClass(ContactHistoryId.class)
+public class ContactHistory extends HistoryEntry implements SqlEntity {
 
-  @Column(nullable = false)
-  VKey<ContactResource> contactRepoId;
+  // Store ContactBase instead of ContactResource so we don't pick up its @Id
+  @Nullable ContactBase contactBase;
+
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getContactRepoId() {
+    return parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setContactRepoId(String contactRepoId) {
+    parent = Key.create(ContactResource.class, contactRepoId);
+  }
 
   @Id
-  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "HistorySequenceGenerator")
   @Column(name = "historyRevisionId")
   @Access(AccessType.PROPERTY)
   @Override
@@ -60,14 +77,98 @@ public class ContactHistory extends HistoryEntry {
     return super.getId();
   }
 
-  /** The state of the {@link ContactBase} object at this point in time. */
-  public ContactBase getContactBase() {
-    return contactBase;
+  /**
+   * The values of all the fields on the {@link ContactBase} object after the action represented by
+   * this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<ContactBase> getContactBase() {
+    return Optional.ofNullable(contactBase);
   }
 
   /** The key to the {@link ContactResource} this is based off of. */
-  public VKey<ContactResource> getContactRepoId() {
-    return contactRepoId;
+  public VKey<ContactResource> getParentVKey() {
+    return VKey.create(ContactResource.class, getContactRepoId());
+  }
+
+  /** Creates a {@link VKey} instance for this entity. */
+  public VKey<ContactHistory> createVKey() {
+    return VKey.create(
+        ContactHistory.class, new ContactHistoryId(getContactRepoId(), getId()), Key.create(this));
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Normally Hibernate would see that the contact fields are all null and would fill contactBase
+    // with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+    if (contactBase != null && contactBase.getContactId() == null) {
+      contactBase = null;
+    }
+  }
+
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
+  /** Class to represent the composite primary key of {@link ContactHistory} entity. */
+  static class ContactHistoryId extends ImmutableObject implements Serializable {
+
+    private String contactRepoId;
+
+    private Long id;
+
+    /** Hibernate requires this default constructor. */
+    private ContactHistoryId() {}
+
+    ContactHistoryId(String contactRepoId, long id) {
+      this.contactRepoId = contactRepoId;
+      this.id = id;
+    }
+
+    /**
+     * Returns the contact repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getContactRepoId() {
+      return contactRepoId;
+    }
+
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
+      return id;
+    }
+
+    /**
+     * Sets the contact repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setContactRepoId(String contactRepoId) {
+      this.contactRepoId = contactRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
+      this.id = id;
+    }
   }
 
   @Override
@@ -88,18 +189,8 @@ public class ContactHistory extends HistoryEntry {
       return this;
     }
 
-    public Builder setContactRepoId(VKey<ContactResource> contactRepoId) {
-      getInstance().contactRepoId = contactRepoId;
-      contactRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
-      return this;
-    }
-
-    // We can remove this once all HistoryEntries are converted to History objects
-    @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().contactRepoId =
-          VKey.create(ContactResource.class, parent.getName(), (Key<ContactResource>) parent);
+    public Builder setContactRepoId(String contactRepoId) {
+      getInstance().parent = Key.create(ContactResource.class, contactRepoId);
       return this;
     }
   }

core/src/main/java/google/registry/model/contact/ContactResource.java

@@ -51,7 +51,6 @@ public class ContactResource extends ContactBase
 
   @Override
   public VKey<ContactResource> createVKey() {
-    // TODO(mmuller): create symmetric keys if we can ever reload both sides.
     return VKey.create(ContactResource.class, getRepoId(), Key.create(this));
   }
 

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -19,6 +19,7 @@ import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
@@ -106,6 +107,32 @@ public class DomainBase extends DomainContent
     return gracePeriods;
   }
 
+  /**
+   * Returns the set of {@link DelegationSignerData} associated with the domain.
+   *
+   * <p>This is the getter method specific for Hibernate to access the field so it is set to
+   * private. The caller can use the public {@link #getDsData()} to get the DS data.
+   *
+   * <p>Note that we need to set `insertable = false, updatable = false` for @JoinColumn, otherwise
+   * Hibernate would try to set the foreign key to null(through an UPDATE TABLE sql) instead of
+   * deleting the whole entry from the table when the {@link DelegationSignerData} is removed from
+   * the set.
+   */
+  @Access(AccessType.PROPERTY)
+  @OneToMany(
+      cascade = {CascadeType.ALL},
+      fetch = FetchType.EAGER,
+      orphanRemoval = true)
+  @JoinColumn(
+      name = "domainRepoId",
+      referencedColumnName = "repoId",
+      insertable = false,
+      updatable = false)
+  @SuppressWarnings("UnusedMethod")
+  private Set<DelegationSignerData> getInternalDelegationSignerData() {
+    return dsData;
+  }
+
   @Override
   public VKey<DomainBase> createVKey() {
     return VKey.create(DomainBase.class, getRepoId(), Key.create(this));

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -35,11 +35,13 @@ import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static google.registry.util.DomainNameUtils.getTldFromDomainName;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Ordering;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Streams;
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
@@ -49,6 +51,7 @@ import google.registry.flows.ResourceFlowUtils;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.launch.LaunchNotice;
 import google.registry.model.domain.rgp.GracePeriodStatus;
@@ -57,6 +60,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registry.Registry;
+import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
@@ -65,7 +69,6 @@ import java.util.HashSet;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.function.Predicate;
 import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -209,6 +212,15 @@ public class DomainContent extends EppResource
   @Column(name = "deletion_poll_message_id")
   VKey<PollMessage.OneTime> deletePollMessage;
 
+  /**
+   * History record for the delete poll message.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Column(name = "deletion_poll_message_history_id")
+  @Ignore
+  Long deletePollMessageHistoryId;
+
   /**
    * The recurring billing event associated with this domain's autorenewals.
    *
@@ -220,6 +232,15 @@ public class DomainContent extends EppResource
   @Column(name = "billing_recurrence_id")
   VKey<BillingEvent.Recurring> autorenewBillingEvent;
 
+  /**
+   * History record for the autorenew billing event.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Column(name = "billing_recurrence_history_id")
+  @Ignore
+  Long autorenewBillingEventHistoryId;
+
   /**
    * The recurring poll message associated with this domain's autorenewals.
    *
@@ -231,6 +252,13 @@ public class DomainContent extends EppResource
   @Column(name = "autorenew_poll_message_id")
   VKey<PollMessage.Autorenew> autorenewPollMessage;
 
+  /**
+   * History record for the autorenew poll message.
+   *
+   * <p>Here so we can restore the original ofy key from sql.
+   */
+  @Ignore Long autorenewPollMessageHistoryId;
+
   /** The unexpired grace periods for this domain (some of which may not be active yet). */
   @Transient Set<GracePeriod> gracePeriods;
 
@@ -284,10 +312,20 @@ public class DomainContent extends EppResource
         nullToEmptyImmutableCopy(gracePeriods).stream()
             .map(gracePeriod -> gracePeriod.cloneWithDomainRepoId(getRepoId()))
             .collect(toImmutableSet());
+
+    // Restore history record ids.
+    autorenewPollMessageHistoryId = getHistoryId(autorenewPollMessage);
+    autorenewBillingEventHistoryId = getHistoryId(autorenewBillingEvent);
+    deletePollMessageHistoryId = getHistoryId(deletePollMessage);
+    dsData =
+        nullToEmptyImmutableCopy(dsData).stream()
+            .map(dsData -> dsData.cloneWithDomainRepoId(getRepoId()))
+            .collect(toImmutableSet());
   }
 
   @PostLoad
-  void postLoad() {
+  @SuppressWarnings("UnusedMethod")
+  private final void postLoad() {
     // Reconstitute the contact list.
     ImmutableSet.Builder<DesignatedContact> contactsBuilder = new ImmutableSet.Builder<>();
 
@@ -306,6 +344,27 @@ public class DomainContent extends EppResource
     }
 
     allContacts = contactsBuilder.build();
+
+    // Reconstitute the composite ofy keys from the SQL data.
+    Key<DomainBase> myKey = Key.create(DomainBase.class, getRepoId());
+    deletePollMessage = restoreOfyFrom(myKey, deletePollMessage, deletePollMessageHistoryId);
+    autorenewBillingEvent =
+        restoreOfyFrom(myKey, autorenewBillingEvent, autorenewBillingEventHistoryId);
+    autorenewPollMessage =
+        restoreOfyFrom(myKey, autorenewPollMessage, autorenewPollMessageHistoryId);
+
+    if (transferData != null) {
+      transferData.restoreOfyKeys(myKey);
+    }
+  }
+
+  public static <T> VKey<T> restoreOfyFrom(Key<DomainBase> domainKey, VKey<T> key, Long historyId) {
+    if (historyId == null) {
+      // This is a legacy key (or a null key, in which case this works too)
+      return VKey.restoreOfyFrom(key, EntityGroupRoot.class, "per-tld");
+    } else {
+      return VKey.restoreOfyFrom(key, domainKey, HistoryEntry.class, historyId);
+    }
   }
 
   public ImmutableSet<String> getSubordinateHosts() {
@@ -387,6 +446,12 @@ public class DomainContent extends EppResource
     this.gracePeriods = gracePeriods;
   }
 
+  // Hibernate needs this in order to populate dsData but no one else should ever use it
+  @SuppressWarnings("UnusedMethod")
+  private void setInternalDelegationSignerData(Set<DelegationSignerData> dsData) {
+    this.dsData = dsData;
+  }
+
   public final String getCurrentSponsorClientId() {
     return getPersistedCurrentSponsorClientId();
   }
@@ -655,13 +720,31 @@ public class DomainContent extends EppResource
             + " use DomainBase instead");
   }
 
+  /**
+   * Obtains a history id from the given key.
+   *
+   * <p>The key must be a composite key either of the form domain-key/history-key/long-event-key or
+   * EntityGroupRoot/long-event-key (for legacy keys). In the latter case or for a null key returns
+   * a history id of null.
+   */
+  public static Long getHistoryId(VKey<?> key) {
+    if (key == null) {
+      return null;
+    }
+    Key<?> parent = key.getOfyKey().getParent();
+    if (parent == null || parent.getKind().equals("EntityGroupRoot")) {
+      return null;
+    }
+    return parent.getId();
+  }
+
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
   static final Predicate<DesignatedContact> IS_REGISTRANT =
       (DesignatedContact contact) -> DesignatedContact.Type.REGISTRANT.equals(contact.type);
 
   /** An override of {@link EppResource#asBuilder} with tighter typing. */
   @Override
-  public Builder asBuilder() {
+  public Builder<? extends DomainContent, ?> asBuilder() {
     return new Builder<>(clone(this));
   }
 
@@ -698,10 +781,16 @@ public class DomainContent extends EppResource
       instance.tld = getTldFromDomainName(instance.fullyQualifiedDomainName);
 
       T newDomain = super.build();
-      // Hibernate throws exception if gracePeriods is null because we enabled all cascadable
-      // operations and orphan removal.
+      // Hibernate throws exception if gracePeriods or dsData is null because we enabled all
+      // cascadable operations and orphan removal.
       newDomain.gracePeriods =
           newDomain.gracePeriods == null ? ImmutableSet.of() : newDomain.gracePeriods;
+      newDomain.dsData =
+          newDomain.dsData == null
+              ? ImmutableSet.of()
+              : newDomain.dsData.stream()
+                  .map(ds -> ds.cloneWithDomainRepoId(instance.getRepoId()))
+                  .collect(toImmutableSet());
       return newDomain;
     }
 
@@ -824,16 +913,19 @@ public class DomainContent extends EppResource
 
     public B setDeletePollMessage(VKey<PollMessage.OneTime> deletePollMessage) {
       getInstance().deletePollMessage = deletePollMessage;
+      getInstance().deletePollMessageHistoryId = getHistoryId(deletePollMessage);
       return thisCastToDerived();
     }
 
     public B setAutorenewBillingEvent(VKey<BillingEvent.Recurring> autorenewBillingEvent) {
       getInstance().autorenewBillingEvent = autorenewBillingEvent;
+      getInstance().autorenewBillingEventHistoryId = getHistoryId(autorenewBillingEvent);
       return thisCastToDerived();
     }
 
     public B setAutorenewPollMessage(VKey<PollMessage.Autorenew> autorenewPollMessage) {
       getInstance().autorenewPollMessage = autorenewPollMessage;
+      getInstance().autorenewPollMessageHistoryId = getHistoryId(autorenewPollMessage);
       return thisCastToDerived();
     }
 

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -16,28 +16,36 @@ package google.registry.model.domain;
 
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.annotation.Ignore;
-import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.host.HostResource;
+import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.io.Serializable;
+import java.util.Optional;
 import java.util.Set;
+import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Id;
 import javax.persistence.IdClass;
 import javax.persistence.Index;
+import javax.persistence.JoinColumn;
 import javax.persistence.JoinTable;
+import javax.persistence.OneToMany;
 import javax.persistence.PostLoad;
 import javax.persistence.Table;
 
@@ -59,11 +67,22 @@ import javax.persistence.Table;
 @EntitySubclass
 @Access(AccessType.FIELD)
 @IdClass(DomainHistoryId.class)
-public class DomainHistory extends HistoryEntry {
-  // Store DomainContent instead of DomainBase so we don't pick up its @Id
-  DomainContent domainContent;
+public class DomainHistory extends HistoryEntry implements SqlEntity {
 
-  @Id String domainRepoId;
+  // Store DomainContent instead of DomainBase so we don't pick up its @Id
+  @Nullable DomainContent domainContent;
+
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    return parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setDomainRepoId(String domainRepoId) {
+    parent = Key.create(DomainBase.class, domainRepoId);
+  }
 
   // We could have reused domainContent.nsHosts here, but Hibernate throws a weird exception after
   // we change to use a composite primary key.
@@ -74,8 +93,51 @@ public class DomainHistory extends HistoryEntry {
   @Column(name = "host_repo_id")
   Set<VKey<HostResource>> nsHosts;
 
+  @Override
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @AttributeOverrides({
+      @AttributeOverride(
+          name = "unit",
+          column = @Column(name = "historyPeriodUnit")),
+      @AttributeOverride(
+          name = "value",
+          column = @Column(name = "historyPeriodValue"))
+  })
+  public Period getPeriod() {
+    return super.getPeriod();
+  }
+
+  /**
+   * For transfers, the id of the other registrar.
+   *
+   * <p>For requests and cancels, the other registrar is the losing party (because the registrar
+   * sending the EPP transfer command is the gaining party). For approves and rejects, the other
+   * registrar is the gaining party.
+   */
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @Column(name = "historyOtherRegistrarId")
+  public String getOtherRegistrarId() {
+    return super.getOtherClientId();
+  }
+
+  /**
+   * Logging field for transaction reporting.
+   *
+   * <p>This will be empty for any DomainHistory/HistoryEntry generated before this field was added,
+   * mid-2017, as well as any action that does not generate billable events (e.g. updates).
+   */
+  @Access(AccessType.PROPERTY)
+  @OneToMany(cascade = {CascadeType.ALL})
+  @JoinColumn(name = "historyRevisionId", referencedColumnName = "historyRevisionId")
+  @JoinColumn(name = "domainRepoId", referencedColumnName = "domainRepoId")
+  @Override
+  public Set<DomainTransactionRecord> getDomainTransactionRecords() {
+    return super.getDomainTransactionRecords();
+  }
+
   @Id
-  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "HistorySequenceGenerator")
   @Column(name = "historyRevisionId")
   @Access(AccessType.PROPERTY)
   @Override
@@ -88,27 +150,49 @@ public class DomainHistory extends HistoryEntry {
     return nsHosts;
   }
 
-  /** The state of the {@link DomainContent} object at this point in time. */
-  public DomainContent getDomainContent() {
-    return domainContent;
+  /**
+   * The values of all the fields on the {@link DomainContent} object after the action represented
+   * by this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<DomainContent> getDomainContent() {
+    return Optional.ofNullable(domainContent);
   }
 
   /** The key to the {@link DomainBase} this is based off of. */
-  public VKey<DomainBase> getDomainRepoId() {
-    return VKey.create(DomainBase.class, domainRepoId, Key.create(DomainBase.class, domainRepoId));
+  public VKey<DomainBase> getParentVKey() {
+    return VKey.create(DomainBase.class, getDomainRepoId());
   }
 
+  /** Creates a {@link VKey} instance for this entity. */
   public VKey<DomainHistory> createVKey() {
-    return VKey.createSql(DomainHistory.class, new DomainHistoryId(domainRepoId, getId()));
+    return VKey.create(
+        DomainHistory.class, new DomainHistoryId(getDomainRepoId(), getId()), Key.create(this));
   }
 
   @PostLoad
   void postLoad() {
     if (domainContent != null) {
       domainContent.nsHosts = nullToEmptyImmutableCopy(nsHosts);
+      // Normally Hibernate would see that the domain fields are all null and would fill
+      // domainContent with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+      if (domainContent.getDomainName() == null) {
+        domainContent = null;
+      } else {
+        if (domainContent.getRepoId() == null) {
+          domainContent = domainContent.asBuilder().setRepoId(parent.getName()).build();
+        }
+      }
     }
   }
 
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
   /** Class to represent the composite primary key of {@link DomainHistory} entity. */
   static class DomainHistoryId extends ImmutableObject implements Serializable {
 
@@ -124,19 +208,45 @@ public class DomainHistory extends HistoryEntry {
       this.id = id;
     }
 
-    String getDomainRepoId() {
+    /**
+     * Returns the domain repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getDomainRepoId() {
       return domainRepoId;
     }
 
-    void setDomainRepoId(String domainRepoId) {
-      this.domainRepoId = domainRepoId;
-    }
-
-    long getId() {
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
       return id;
     }
 
-    void setId(long id) {
+    /**
+     * Sets the domain repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setDomainRepoId(String domainRepoId) {
+      this.domainRepoId = domainRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
       this.id = id;
     }
   }
@@ -163,17 +273,8 @@ public class DomainHistory extends HistoryEntry {
     }
 
     public Builder setDomainRepoId(String domainRepoId) {
-      getInstance().domainRepoId = domainRepoId;
       getInstance().parent = Key.create(DomainBase.class, domainRepoId);
       return this;
     }
-
-    // We can remove this once all HistoryEntries are converted to History objects
-    @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().domainRepoId = parent.getName();
-      return this;
-    }
   }
 }

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -22,6 +22,7 @@ import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import javax.annotation.Nullable;
 import javax.persistence.Entity;
 import javax.persistence.Index;
@@ -37,7 +38,7 @@ import org.joda.time.DateTime;
 @Embed
 @Entity
 @Table(indexes = @Index(columnList = "domainRepoId"))
-public class GracePeriod extends GracePeriodBase {
+public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntity {
 
   private static GracePeriod createInternal(
       GracePeriodStatus type,

core/src/main/java/google/registry/model/domain/Period.java

@@ -31,9 +31,9 @@ public class Period extends ImmutableObject {
   @XmlAttribute
   Unit unit;
 
-  @XmlValue
-  Integer value;
+  @XmlValue Integer value;
 
+  @Enumerated(EnumType.STRING)
   public Unit getUnit() {
     return unit;
   }
@@ -42,6 +42,18 @@ public class Period extends ImmutableObject {
     return value;
   }
 
+  /** This method exists solely to satisfy Hibernate. Use {@link #create(int, Unit)} instead. */
+  @SuppressWarnings("UnusedMethod")
+  private void setUnit(Unit unit) {
+    this.unit = unit;
+  }
+
+  /** This method exists solely to satisfy Hibernate. Use {@link #create(int, Unit)} instead. */
+  @SuppressWarnings("UnusedMethod")
+  private void setValue(Integer value) {
+    this.value = value;
+  }
+
   /** The unit enum. */
   public enum Unit {
     @XmlEnumValue("y")

core/src/main/java/google/registry/model/domain/secdns/DelegationSignerData.java

@@ -14,10 +14,22 @@
 
 package google.registry.model.domain.secdns;
 
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+
 import com.googlecode.objectify.annotation.Embed;
+import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.domain.secdns.DelegationSignerData.DelegationSignerDataId;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
+import java.io.Serializable;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.IdClass;
+import javax.persistence.Index;
+import javax.persistence.Table;
 import javax.xml.bind.DatatypeConverter;
 import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlTransient;
 import javax.xml.bind.annotation.XmlType;
 import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
 import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
@@ -30,19 +42,26 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
  */
 @Embed
 @XmlType(name = "dsData")
-@javax.persistence.Entity
-public class DelegationSignerData extends ImmutableObject {
+@Entity
+@Table(indexes = @Index(columnList = "domainRepoId"))
+@IdClass(DelegationSignerDataId.class)
+public class DelegationSignerData extends ImmutableObject implements DatastoreAndSqlEntity {
 
   private DelegationSignerData() {}
 
+  @Ignore @XmlTransient @javax.persistence.Id String domainRepoId;
+
   /** The identifier for this particular key in the domain. */
-  @javax.persistence.Id int keyTag;
+  @javax.persistence.Id
+  @Column(nullable = false)
+  int keyTag;
 
   /**
    * The algorithm used by this key.
    *
    * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.1">RFC 4034 Appendix A.1</a>
    */
+  @Column(nullable = false)
   @XmlElement(name = "alg")
   int algorithm;
 
@@ -51,6 +70,7 @@ public class DelegationSignerData extends ImmutableObject {
    *
    * @see <a href="http://tools.ietf.org/html/rfc4034#appendix-A.2">RFC 4034 Appendix A.2</a>
    */
+  @Column(nullable = false)
   int digestType;
 
   /**
@@ -58,6 +78,7 @@ public class DelegationSignerData extends ImmutableObject {
    *
    * @see <a href="http://tools.ietf.org/html/rfc4034#section-5.1.4">RFC 4034 Section 5.1.4</a>
    */
+  @Column(nullable = false)
   @XmlJavaTypeAdapter(HexBinaryAdapter.class)
   byte[] digest;
 
@@ -81,16 +102,34 @@ public class DelegationSignerData extends ImmutableObject {
     return digest == null ? "" : DatatypeConverter.printHexBinary(digest);
   }
 
+  public DelegationSignerData cloneWithDomainRepoId(String domainRepoId) {
+    DelegationSignerData clone = clone(this);
+    clone.domainRepoId = checkArgumentNotNull(domainRepoId);
+    return clone;
+  }
+
+  public DelegationSignerData cloneWithoutDomainRepoId() {
+    DelegationSignerData clone = clone(this);
+    clone.domainRepoId = null;
+    return clone;
+  }
+
   public static DelegationSignerData create(
-      int keyTag, int algorithm, int digestType, byte[] digest) {
+      int keyTag, int algorithm, int digestType, byte[] digest, String domainRepoId) {
     DelegationSignerData instance = new DelegationSignerData();
     instance.keyTag = keyTag;
     instance.algorithm = algorithm;
     instance.digestType = digestType;
     instance.digest = digest;
+    instance.domainRepoId = domainRepoId;
     return instance;
   }
 
+  public static DelegationSignerData create(
+      int keyTag, int algorithm, int digestType, byte[] digest) {
+    return create(keyTag, algorithm, digestType, digest, null);
+  }
+
   public static DelegationSignerData create(
       int keyTag, int algorithm, int digestType, String digestAsHex) {
     return create(keyTag, algorithm, digestType, DatatypeConverter.parseHexBinary(digestAsHex));
@@ -106,4 +145,20 @@ public class DelegationSignerData extends ImmutableObject {
         "%d %d %d %s",
         this.keyTag, this.algorithm, this.digestType, DatatypeConverter.printHexBinary(digest));
   }
+
+  static class DelegationSignerDataId extends ImmutableObject implements Serializable {
+    String domainRepoId;
+    int keyTag;
+
+    private DelegationSignerDataId() {}
+
+    private DelegationSignerDataId(String domainRepoId, int keyTag) {
+      this.domainRepoId = domainRepoId;
+      this.keyTag = keyTag;
+    }
+
+    public static DelegationSignerDataId create(String domainRepoId, int keyTag) {
+      return new DelegationSignerDataId(checkArgumentNotNull(domainRepoId), keyTag);
+    }
+  }
 }

core/src/main/java/google/registry/model/eppcommon/StatusValue.java

@@ -20,8 +20,10 @@ import static com.google.common.base.Strings.nullToEmpty;
 
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactBase;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
 import google.registry.model.host.HostBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.translators.EnumToAttributeAdapter.EppEnum;
@@ -128,9 +130,15 @@ public enum StatusValue implements EppEnum {
 
   /** Enum to help clearly list which resource types a status value is allowed to be present on. */
   private enum AllowedOn {
-    ALL(ContactResource.class, DomainBase.class, HostBase.class, HostResource.class),
+    ALL(
+        ContactBase.class,
+        ContactResource.class,
+        DomainContent.class,
+        DomainBase.class,
+        HostBase.class,
+        HostResource.class),
     NONE,
-    DOMAINS(DomainBase.class);
+    DOMAINS(DomainContent.class, DomainBase.class);
 
     private final ImmutableSet<Class<? extends EppResource>> classes;
 

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -14,18 +14,25 @@
 
 package google.registry.model.host;
 
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
-import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.host.HostHistory.HostHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.annotation.Nullable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
 
 /**
  * A persisted history entry representing an EPP modification to a host.
@@ -45,16 +52,25 @@ import javax.persistence.Id;
     })
 @EntitySubclass
 @Access(AccessType.FIELD)
-public class HostHistory extends HistoryEntry {
+@IdClass(HostHistoryId.class)
+public class HostHistory extends HistoryEntry implements SqlEntity {
 
   // Store HostBase instead of HostResource so we don't pick up its @Id
-  HostBase hostBase;
-
-  @Column(nullable = false)
-  VKey<HostResource> hostRepoId;
+  @Nullable HostBase hostBase;
+
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getHostRepoId() {
+    return parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setHostRepoId(String hostRepoId) {
+    parent = Key.create(HostResource.class, hostRepoId);
+  }
 
   @Id
-  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "HistorySequenceGenerator")
   @Column(name = "historyRevisionId")
   @Access(AccessType.PROPERTY)
   @Override
@@ -62,14 +78,98 @@ public class HostHistory extends HistoryEntry {
     return super.getId();
   }
 
-  /** The state of the {@link HostBase} object at this point in time. */
-  public HostBase getHostBase() {
-    return hostBase;
+  /**
+   * The values of all the fields on the {@link HostBase} object after the action represented by
+   * this history object was executed.
+   *
+   * <p>Will be absent for objects created prior to the Registry 3.0 SQL migration.
+   */
+  public Optional<HostBase> getHostBase() {
+    return Optional.ofNullable(hostBase);
   }
 
   /** The key to the {@link google.registry.model.host.HostResource} this is based off of. */
-  public VKey<HostResource> getHostRepoId() {
-    return hostRepoId;
+  public VKey<HostResource> getParentVKey() {
+    return VKey.create(HostResource.class, getHostRepoId());
+  }
+
+  /** Creates a {@link VKey} instance for this entity. */
+  public VKey<HostHistory> createVKey() {
+    return VKey.create(
+        HostHistory.class, new HostHistoryId(getHostRepoId(), getId()), Key.create(this));
+  }
+
+  @PostLoad
+  void postLoad() {
+    // Normally Hibernate would see that the host fields are all null and would fill hostBase
+    // with a null object. Unfortunately, the updateTimestamp is never null in SQL.
+    if (hostBase != null && hostBase.getHostName() == null) {
+      hostBase = null;
+    }
+  }
+
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
+  /** Class to represent the composite primary key of {@link HostHistory} entity. */
+  static class HostHistoryId extends ImmutableObject implements Serializable {
+
+    private String hostRepoId;
+
+    private Long id;
+
+    /** Hibernate requires this default constructor. */
+    private HostHistoryId() {}
+
+    HostHistoryId(String hostRepoId, long id) {
+      this.hostRepoId = hostRepoId;
+      this.id = id;
+    }
+
+    /**
+     * Returns the host repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private String getHostRepoId() {
+      return hostRepoId;
+    }
+
+    /**
+     * Returns the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate.
+     */
+    @SuppressWarnings("unused")
+    private long getId() {
+      return id;
+    }
+
+    /**
+     * Sets the host repository id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setHostRepoId(String hostRepoId) {
+      this.hostRepoId = hostRepoId;
+    }
+
+    /**
+     * Sets the history revision id.
+     *
+     * <p>This method is private because it is only used by Hibernate and should not be used
+     * externally to keep immutability.
+     */
+    @SuppressWarnings("unused")
+    private void setId(long id) {
+      this.id = id;
+    }
   }
 
   @Override
@@ -90,18 +190,8 @@ public class HostHistory extends HistoryEntry {
       return this;
     }
 
-    public Builder setHostRepoId(VKey<HostResource> hostRepoId) {
-      getInstance().hostRepoId = hostRepoId;
-      hostRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
-      return this;
-    }
-
-    // We can remove this once all HistoryEntries are converted to History objects
-    @Override
-    public Builder setParent(Key<? extends EppResource> parent) {
-      super.setParent(parent);
-      getInstance().hostRepoId =
-          VKey.create(HostResource.class, parent.getName(), (Key<HostResource>) parent);
+    public Builder setHostRepoId(String hostRepoId) {
+      getInstance().parent = Key.create(HostResource.class, hostRepoId);
       return this;
     }
   }

core/src/main/java/google/registry/model/host/HostResource.java

@@ -32,7 +32,7 @@ import javax.persistence.AccessType;
  */
 @ReportedOn
 @Entity
-@javax.persistence.Entity
+@javax.persistence.Entity(name = "Host")
 @ExternalMessagingName("host")
 @WithStringVKey
 @Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property)

core/src/main/java/google/registry/model/index/EppResourceIndex.java

@@ -17,6 +17,7 @@ package google.registry.model.index;
 import static google.registry.util.TypeUtils.instantiate;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -25,11 +26,13 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.EppResource;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /** An index that allows for quick enumeration of all EppResource entities (e.g. via map reduce). */
 @ReportedOn
 @Entity
-public class EppResourceIndex extends BackupGroupRoot {
+public class EppResourceIndex extends BackupGroupRoot implements DatastoreEntity {
 
   @Id
   String id;
@@ -74,4 +77,9 @@ public class EppResourceIndex extends BackupGroupRoot {
   public static <T extends EppResource> EppResourceIndex create(Key<T> resourceKey) {
     return create(EppResourceIndexBucket.getBucketKey(resourceKey), resourceKey);
   }
+
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not relevant in SQL
+  }
 }

core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java

@@ -24,16 +24,23 @@ import com.googlecode.objectify.annotation.Id;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.VirtualEntity;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 
 /** A virtual entity to represent buckets to which EppResourceIndex objects are randomly added. */
 @Entity
 @VirtualEntity
-public class EppResourceIndexBucket extends ImmutableObject {
+public class EppResourceIndexBucket extends ImmutableObject implements DatastoreEntity {
 
   @SuppressWarnings("unused")
   @Id
   private long bucketId;
 
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not relevant in SQL
+  }
+
   /**
    * Deterministic function that returns a bucket id based on the resource's roid.
    * NB: At the moment, nothing depends on this being deterministic, so we have the ability to

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -43,6 +43,8 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
 import java.util.Optional;
@@ -61,28 +63,44 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   /** The {@link ForeignKeyIndex} type for {@link ContactResource} entities. */
   @ReportedOn
   @Entity
-  public static class ForeignKeyContactIndex extends ForeignKeyIndex<ContactResource> {}
+  public static class ForeignKeyContactIndex extends ForeignKeyIndex<ContactResource>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }
 
   /** The {@link ForeignKeyIndex} type for {@link DomainBase} entities. */
   @ReportedOn
   @Entity
-  public static class ForeignKeyDomainIndex extends ForeignKeyIndex<DomainBase> {}
+  public static class ForeignKeyDomainIndex extends ForeignKeyIndex<DomainBase>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }
 
   /** The {@link ForeignKeyIndex} type for {@link HostResource} entities. */
   @ReportedOn
   @Entity
-  public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource> {}
+  public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }
 
-  static final ImmutableMap<
-          Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
+  static final ImmutableMap<Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
       RESOURCE_CLASS_TO_FKI_CLASS =
           ImmutableMap.of(
               ContactResource.class, ForeignKeyContactIndex.class,
               DomainBase.class, ForeignKeyDomainIndex.class,
               HostResource.class, ForeignKeyHostIndex.class);
 
-  @Id
-  String foreignKey;
+  @Id String foreignKey;
 
   /**
    * The deletion time of this {@link ForeignKeyIndex}.
@@ -90,8 +108,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * <p>This will generally be equal to the deletion time of {@link #topReference}. However, in the
    * case of a {@link HostResource} that was renamed, this field will hold the time of the rename.
    */
-  @Index
-  DateTime deletionTime;
+  @Index DateTime deletionTime;
 
   /**
    * The referenced resource.

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -101,22 +101,22 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public void saveNew(Object entity) {
+  public void insert(Object entity) {
     saveEntity(entity);
   }
 
   @Override
-  public void saveAllNew(ImmutableCollection<?> entities) {
+  public void insertAll(ImmutableCollection<?> entities) {
     getOfy().save().entities(entities);
   }
 
   @Override
-  public void saveNewOrUpdate(Object entity) {
+  public void put(Object entity) {
     saveEntity(entity);
   }
 
   @Override
-  public void saveNewOrUpdateAll(ImmutableCollection<?> entities) {
+  public void putAll(ImmutableCollection<?> entities) {
     getOfy().save().entities(entities);
   }
 
@@ -131,12 +131,12 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public boolean checkExists(Object entity) {
+  public boolean exists(Object entity) {
     return getOfy().load().key(Key.create(entity)).now() != null;
   }
 
   @Override
-  public <T> boolean checkExists(VKey<T> key) {
+  public <T> boolean exists(VKey<T> key) {
     return loadNullable(key) != null;
   }
 

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -27,14 +27,18 @@ import com.googlecode.objectify.annotation.EntitySubclass;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainRenewData;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
+import google.registry.model.host.HostResource;
 import google.registry.model.poll.PendingActionNotificationResponse.ContactPendingActionNotificationResponse;
 import google.registry.model.poll.PendingActionNotificationResponse.DomainPendingActionNotificationResponse;
 import google.registry.model.poll.PendingActionNotificationResponse.HostPendingActionNotificationResponse;
@@ -45,6 +49,7 @@ import google.registry.model.transfer.TransferResponse.ContactTransferResponse;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.List;
 import java.util.Optional;
 import javax.persistence.AttributeOverride;
@@ -53,10 +58,9 @@ import javax.persistence.Column;
 import javax.persistence.DiscriminatorColumn;
 import javax.persistence.DiscriminatorValue;
 import javax.persistence.Embedded;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
+import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
@@ -92,12 +96,11 @@ import org.joda.time.DateTime;
       @javax.persistence.Index(columnList = "eventTime")
     })
 public abstract class PollMessage extends ImmutableObject
-    implements Buildable, TransferServerApproveEntity {
+    implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity {
 
   /** Entity id. */
   @Id
   @javax.persistence.Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
   @Column(name = "poll_message_id")
   Long id;
 
@@ -123,11 +126,11 @@ public abstract class PollMessage extends ImmutableObject
 
   @Ignore String hostRepoId;
 
-  @Ignore Long domainRevisionId;
+  @Ignore Long domainHistoryRevisionId;
 
-  @Ignore Long contactRevisionId;
+  @Ignore Long contactHistoryRevisionId;
 
-  @Ignore Long hostRevisionId;
+  @Ignore Long hostHistoryRevisionId;
 
   public Key<HistoryEntry> getParentKey() {
     return parent;
@@ -151,6 +154,34 @@ public abstract class PollMessage extends ImmutableObject
 
   public abstract ImmutableList<ResponseData> getResponseData();
 
+  @PostLoad
+  void postLoad() {
+    if (domainRepoId != null) {
+      parent =
+          Key.create(
+              Key.create(DomainBase.class, domainRepoId),
+              HistoryEntry.class,
+              domainHistoryRevisionId);
+    } else if (contactRepoId != null) {
+      parent =
+          Key.create(
+              Key.create(ContactResource.class, contactRepoId),
+              HistoryEntry.class,
+              contactHistoryRevisionId);
+    } else if (hostHistoryRevisionId != null) {
+      parent =
+          Key.create(
+              Key.create(HostResource.class, hostRepoId),
+              HistoryEntry.class,
+              hostHistoryRevisionId);
+    }
+  }
+
+  @OnLoad
+  void onLoad() {
+    setSqlForeignKeys(this);
+  }
+
   @Override
   public abstract VKey<? extends PollMessage> createVKey();
 
@@ -216,10 +247,30 @@ public abstract class PollMessage extends ImmutableObject
       checkArgumentNotNull(instance.clientId, "clientId must be specified");
       checkArgumentNotNull(instance.eventTime, "eventTime must be specified");
       checkArgumentNotNull(instance.parent, "parent must be specified");
+      checkArgumentNotNull(instance.parent.getParent(), "parent.getParent() must be specified");
+      setSqlForeignKeys(instance);
       return super.build();
     }
   }
 
+  private static void setSqlForeignKeys(PollMessage pollMessage) {
+    String grandparentKind = pollMessage.parent.getParent().getKind();
+    String repoId = pollMessage.parent.getParent().getName();
+    long historyRevisionId = pollMessage.parent.getId();
+    if (Key.getKind(DomainBase.class).equals(grandparentKind)) {
+      pollMessage.domainRepoId = repoId;
+      pollMessage.domainHistoryRevisionId = historyRevisionId;
+    } else if (Key.getKind(ContactResource.class).equals(grandparentKind)) {
+      pollMessage.contactRepoId = repoId;
+      pollMessage.contactHistoryRevisionId = historyRevisionId;
+    } else if (Key.getKind(HostResource.class).equals(grandparentKind)) {
+      pollMessage.hostRepoId = repoId;
+      pollMessage.hostHistoryRevisionId = historyRevisionId;
+    } else {
+      throw new IllegalArgumentException("Unknown grandparent kind: " + grandparentKind);
+    }
+  }
+
   /**
    * A one-time poll message.
    *

core/src/main/java/google/registry/model/registry/Registry.java

@@ -62,6 +62,7 @@ import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Fee;
 import google.registry.model.registry.label.PremiumList;
 import google.registry.model.registry.label.ReservedList;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.util.Idn;
 import java.util.Map;
 import java.util.Optional;
@@ -70,6 +71,13 @@ import java.util.concurrent.ExecutionException;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.PostLoad;
+import javax.persistence.Transient;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -78,21 +86,31 @@ import org.joda.time.Duration;
 /** Persisted per-TLD configuration data. */
 @ReportedOn
 @Entity
-public class Registry extends ImmutableObject implements Buildable {
+@javax.persistence.Entity(name = "Tld")
+public class Registry extends ImmutableObject implements Buildable, DatastoreAndSqlEntity {
 
-  @Parent Key<EntityGroupRoot> parent = getCrossTldKey();
+  @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
   /**
    * The canonical string representation of the TLD associated with this {@link Registry}, which is
    * the standard ASCII for regular TLDs and punycoded ASCII for IDN TLDs.
    */
-  @Id String tldStrId;
+  @Id
+  @javax.persistence.Id
+  @Column(name = "tld_name", nullable = false)
+  String tldStrId;
 
   /**
    * A duplicate of {@link #tldStrId}, to simplify BigQuery reporting since the id field becomes
    * {@code __key__.name} rather than being exported as a named field.
    */
-  String tldStr;
+  @Transient String tldStr;
+
+  /** Sets the Datastore specific field, tldStr, when the entity is loaded from Cloud SQL */
+  @PostLoad
+  void postLoad() {
+    tldStr = tldStrId;
+  }
 
   /** The suffix that identifies roids as belonging to this specific tld, e.g. -HOW for .how. */
   String roidSuffix;
@@ -116,6 +134,7 @@ public class Registry extends ImmutableObject implements Buildable {
   public static final Money DEFAULT_RENEW_BILLING_COST = Money.of(USD, 8);
   public static final Money DEFAULT_RESTORE_BILLING_COST = Money.of(USD, 100);
   public static final Money DEFAULT_SERVER_STATUS_CHANGE_BILLING_COST = Money.of(USD, 20);
+  public static final Money DEFAULT_REGISTRY_LOCK_OR_UNLOCK_BILLING_COST = Money.of(USD, 0);
 
   /** The type of TLD, which determines things like backups and escrow policy. */
   public enum TldType {
@@ -289,6 +308,7 @@ public class Registry extends ImmutableObject implements Buildable {
    * <p>All entries of this list must be valid keys for the map of {@code DnsWriter}s injected by
    * <code>@Inject Map<String, DnsWriter></code>
    */
+  @Column(nullable = false)
   Set<String> dnsWriters;
 
   /**
@@ -312,6 +332,7 @@ public class Registry extends ImmutableObject implements Buildable {
    * <p>Failure to do so can result in parallel writes to the {@link
    * google.registry.dns.writer.DnsWriter}, which may be dangerous depending on your implementation.
    */
+  @Column(nullable = false)
   int numDnsPublishLocks;
 
   /** Updates an unset numDnsPublishLocks (0) to the standard default of 1. */
@@ -327,6 +348,7 @@ public class Registry extends ImmutableObject implements Buildable {
    * <p>This will be equal to {@link #tldStr} for ASCII TLDs, but will be non-ASCII for IDN TLDs. We
    * store this in a field so that it will be retained upon import into BigQuery.
    */
+  @Column(nullable = false)
   String tldUnicode;
 
   /**
@@ -334,31 +356,37 @@ public class Registry extends ImmutableObject implements Buildable {
    *
    * <p>This is optional; if not configured, then information won't be exported for this TLD.
    */
-  String driveFolderId;
+  @Nullable String driveFolderId;
 
   /** The type of the TLD, whether it's real or for testing. */
+  @Column(nullable = false)
+  @Enumerated(EnumType.STRING)
   TldType tldType = TldType.REAL;
 
   /**
-   * Whether to disable invoicing for a {@link TldType#REAL} TLD.
+   * Whether to enable invoicing for this TLD.
    *
-   * <p>Note that invoicing is always disabled for {@link TldType#TEST} TLDs. Setting this field has
-   * no effect for {@link TldType#TEST} TLDs.
+   * <p>Note that this boolean is the sole determiner on whether invoices should be generated for a
+   * TLD. This applies to {@link TldType#TEST} TLDs as well.
    */
-  boolean disableInvoicing = false;
+  @Column(nullable = false)
+  boolean invoicingEnabled = false;
 
   /**
    * A property that transitions to different TldStates at different times. Stored as a list of
    * TldStateTransition embedded objects using the @Mapify annotation.
    */
+  @Column(nullable = false)
   @Mapify(TimedTransitionProperty.TimeMapper.class)
   TimedTransitionProperty<TldState, TldStateTransition> tldStateTransitions =
       TimedTransitionProperty.forMapify(DEFAULT_TLD_STATE, TldStateTransition.class);
 
   /** An automatically managed creation timestamp. */
+  @Column(nullable = false)
   CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /** The set of reserved lists that are applicable to this registry. */
+  @Column(name = "reserved_list_names", nullable = false)
   Set<Key<ReservedList>> reservedLists;
 
   /** Retrieves an ImmutableSet of all ReservedLists associated with this tld. */
@@ -367,12 +395,15 @@ public class Registry extends ImmutableObject implements Buildable {
   }
 
   /** The static {@link PremiumList} for this TLD, if there is one. */
+  @Column(name = "premium_list_name", nullable = true)
   Key<PremiumList> premiumList;
 
   /** Should RDE upload a nightly escrow deposit for this TLD? */
+  @Column(nullable = false)
   boolean escrowEnabled = DEFAULT_ESCROW_ENABLED;
 
   /** Whether the pull queue that writes to authoritative DNS is paused for this TLD. */
+  @Column(nullable = false)
   boolean dnsPaused = DEFAULT_DNS_PAUSED;
 
   /**
@@ -381,41 +412,85 @@ public class Registry extends ImmutableObject implements Buildable {
    * <p>Domain deletes are free and effective immediately so long as they take place within this
    * amount of time following creation.
    */
+  @Column(nullable = false)
   Duration addGracePeriodLength = DEFAULT_ADD_GRACE_PERIOD;
 
   /** The length of the anchor tenant add grace period for this TLD. */
+  @Column(nullable = false)
   Duration anchorTenantAddGracePeriodLength = DEFAULT_ANCHOR_TENANT_ADD_GRACE_PERIOD;
 
   /** The length of the auto renew grace period for this TLD. */
+  @Column(nullable = false)
   Duration autoRenewGracePeriodLength = DEFAULT_AUTO_RENEW_GRACE_PERIOD;
 
   /** The length of the redemption grace period for this TLD. */
+  @Column(nullable = false)
   Duration redemptionGracePeriodLength = DEFAULT_REDEMPTION_GRACE_PERIOD;
 
   /** The length of the renew grace period for this TLD. */
+  @Column(nullable = false)
   Duration renewGracePeriodLength = DEFAULT_RENEW_GRACE_PERIOD;
 
   /** The length of the transfer grace period for this TLD. */
+  @Column(nullable = false)
   Duration transferGracePeriodLength = DEFAULT_TRANSFER_GRACE_PERIOD;
 
   /** The length of time before a transfer is automatically approved for this TLD. */
+  @Column(nullable = false)
   Duration automaticTransferLength = DEFAULT_AUTOMATIC_TRANSFER_LENGTH;
 
   /** The length of time a domain spends in the non-redeemable pending delete phase for this TLD. */
+  @Column(nullable = false)
   Duration pendingDeleteLength = DEFAULT_PENDING_DELETE_LENGTH;
 
   /** The currency unit for all costs associated with this TLD. */
+  @Column(nullable = false)
   CurrencyUnit currency = DEFAULT_CURRENCY;
 
   /** The per-year billing cost for registering a new domain name. */
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "money.amount",
+        column = @Column(name = "create_billing_cost_amount")),
+    @AttributeOverride(
+        name = "money.currency",
+        column = @Column(name = "create_billing_cost_currency"))
+  })
   Money createBillingCost = DEFAULT_CREATE_BILLING_COST;
 
   /** The one-time billing cost for restoring a domain name from the redemption grace period. */
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "money.amount",
+        column = @Column(name = "restore_billing_cost_amount")),
+    @AttributeOverride(
+        name = "money.currency",
+        column = @Column(name = "restore_billing_cost_currency"))
+  })
   Money restoreBillingCost = DEFAULT_RESTORE_BILLING_COST;
 
   /** The one-time billing cost for changing the server status (i.e. lock). */
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "money.amount",
+        column = @Column(name = "server_status_change_billing_cost_amount")),
+    @AttributeOverride(
+        name = "money.currency",
+        column = @Column(name = "server_status_change_billing_cost_currency"))
+  })
   Money serverStatusChangeBillingCost = DEFAULT_SERVER_STATUS_CHANGE_BILLING_COST;
 
+  /** The one-time billing cost for a registry lock/unlock action initiated by a registrar. */
+  @AttributeOverrides({
+      @AttributeOverride(
+          name = "money.amount",
+          column = @Column(name = "registry_lock_or_unlock_cost_amount")),
+      @AttributeOverride(
+          name = "money.currency",
+          column = @Column(name = "registry_lock_or_unlock_cost_currency"))
+  })
+  Money registryLockOrUnlockBillingCost = DEFAULT_REGISTRY_LOCK_OR_UNLOCK_BILLING_COST;
+
   /**
    * A property that transitions to different renew billing costs at different times. Stored as a
    * list of BillingCostTransition embedded objects using the @Mapify annotation.
@@ -424,11 +499,13 @@ public class Registry extends ImmutableObject implements Buildable {
    * name. This cost is also used to compute costs for transfers, since each transfer includes a
    * renewal to ensure transfers have a cost.
    */
+  @Column(nullable = false)
   @Mapify(TimedTransitionProperty.TimeMapper.class)
   TimedTransitionProperty<Money, BillingCostTransition> renewBillingCostTransitions =
       TimedTransitionProperty.forMapify(DEFAULT_RENEW_BILLING_COST, BillingCostTransition.class);
 
   /** A property that tracks the EAP fee schedule (if any) for the TLD. */
+  @Column(nullable = false)
   @Mapify(TimedTransitionProperty.TimeMapper.class)
   TimedTransitionProperty<Money, BillingCostTransition> eapFeeSchedule =
       TimedTransitionProperty.forMapify(DEFAULT_EAP_BILLING_COST, BillingCostTransition.class);
@@ -437,13 +514,14 @@ public class Registry extends ImmutableObject implements Buildable {
   String lordnUsername;
 
   /** The end of the claims period (at or after this time, claims no longer applies). */
+  @Column(nullable = false)
   DateTime claimsPeriodEnd = END_OF_TIME;
 
   /** An allow list of clients allowed to be used on domains on this TLD (ignored if empty). */
-  Set<String> allowedRegistrantContactIds;
+  @Nullable Set<String> allowedRegistrantContactIds;
 
   /** An allow list of hosts allowed to be used on domains on this TLD (ignored if empty). */
-  Set<String> allowedFullyQualifiedHostNames;
+  @Nullable Set<String> allowedFullyQualifiedHostNames;
 
   public String getTldStr() {
     return tldStr;
@@ -566,6 +644,11 @@ public class Registry extends ImmutableObject implements Buildable {
     return serverStatusChangeBillingCost;
   }
 
+  /** Returns the cost of a registry lock/unlock. */
+  public Money getRegistryLockOrUnlockBillingCost() {
+    return registryLockOrUnlockBillingCost;
+  }
+
   public ImmutableSortedMap<DateTime, TldState> getTldStateTransitions() {
     return tldStateTransitions.toValueMap();
   }
@@ -646,8 +729,8 @@ public class Registry extends ImmutableObject implements Buildable {
       return this;
     }
 
-    public Builder setDisableInvoicing(boolean disableInvoicing) {
-      getInstance().disableInvoicing = disableInvoicing;
+    public Builder setInvoicingEnabled(boolean invoicingEnabled) {
+      getInstance().invoicingEnabled = invoicingEnabled;
       return this;
     }
 
@@ -867,6 +950,12 @@ public class Registry extends ImmutableObject implements Buildable {
       return this;
     }
 
+    public Builder setRegistryLockOrUnlockBillingCost(Money amount) {
+      checkArgument(amount.isPositiveOrZero(), "Registry lock/unlock cost cannot be negative");
+      getInstance().registryLockOrUnlockBillingCost = amount;
+      return this;
+    }
+
     public Builder setLordnUsername(String username) {
       getInstance().lordnUsername = username;
       return this;
@@ -918,6 +1007,9 @@ public class Registry extends ImmutableObject implements Buildable {
       checkArgument(
           instance.getServerStatusChangeCost().getCurrencyUnit().equals(instance.currency),
           "Server status change cost must be in the registry's currency");
+      checkArgument(
+          instance.getRegistryLockOrUnlockBillingCost().getCurrencyUnit().equals(instance.currency),
+          "Registry lock/unlock cost must be in the registry's currency");
       Predicate<Money> currencyCheck =
           (Money money) -> money.getCurrencyUnit().equals(instance.currency);
       checkArgument(

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -114,7 +114,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   /** Virtual parent entity for premium list entry entities associated with a single revision. */
   @ReportedOn
   @Entity
-  public static class PremiumListRevision extends ImmutableObject {
+  public static class PremiumListRevision extends ImmutableObject implements DatastoreEntity {
 
     @Parent Key<PremiumList> parent;
 
@@ -171,6 +171,11 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
       }
       return revision;
     }
+
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not persisted in SQL
+    }
   }
 
   /**

core/src/main/java/google/registry/model/registry/label/ReservedListDualWriteDao.java

@@ -43,7 +43,7 @@ public class ReservedListDualWriteDao {
 
   /** Persist a new reserved list to Cloud SQL. */
   public static void save(ReservedList reservedList) {
-    ofyTm().transact(() -> ofyTm().saveNewOrUpdate(reservedList));
+    ofyTm().transact(() -> ofyTm().put(reservedList));
     try {
       logger.atInfo().log("Saving reserved list %s to Cloud SQL", reservedList.getName());
       ReservedListSqlDao.save(reservedList);

core/src/main/java/google/registry/model/registry/label/ReservedListSqlDao.java

@@ -31,7 +31,7 @@ public class ReservedListSqlDao {
   /** Persist a new reserved list to Cloud SQL. */
   public static void save(ReservedList reservedList) {
     checkArgumentNotNull(reservedList, "Must specify reservedList");
-    jpaTm().transact(() -> jpaTm().saveNew(reservedList));
+    jpaTm().transact(() -> jpaTm().insert(reservedList));
   }
 
   /**

core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java

@@ -19,8 +19,17 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.annotation.Embed;
+import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
 import org.joda.time.DateTime;
 
 /**
@@ -34,9 +43,17 @@ import org.joda.time.DateTime;
  * uses HistoryEntry.otherClientId because the losing party in a transfer is always the otherClient.
  */
 @Embed
-public class DomainTransactionRecord extends ImmutableObject implements Buildable {
+@Entity
+public class DomainTransactionRecord extends ImmutableObject
+    implements Buildable, DatastoreAndSqlEntity {
+
+  @Id
+  @Ignore
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  Long id;
 
   /** The TLD this record operates on. */
+  @Column(nullable = false)
   String tld;
 
   /**
@@ -50,9 +67,12 @@ public class DomainTransactionRecord extends ImmutableObject implements Buildabl
    *     href="https://www.icann.org/resources/unthemed-pages/registry-agmt-appc-10-2001-05-11-en">
    *     Grace period spec</a>
    */
+  @Column(nullable = false)
   DateTime reportingTime;
 
   /** The transaction report field we add reportAmount to for this registrar. */
+  @Column(nullable = false)
+  @Enumerated(value = EnumType.STRING)
   TransactionReportField reportField;
 
   /**
@@ -67,6 +87,7 @@ public class DomainTransactionRecord extends ImmutableObject implements Buildabl
    * original SUCCESSFUL transfer counters. Finally, if we explicitly allow a transfer, the report
    * amount is 0, as we've already counted the transfer in the original request.
    */
+  @Column(nullable = false)
   Integer reportAmount;
 
   /**

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -18,6 +18,7 @@ import static com.googlecode.objectify.Key.getKind;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -40,6 +41,8 @@ import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.persistence.Access;
@@ -59,7 +62,7 @@ import org.joda.time.DateTime;
 @MappedSuperclass
 @WithStringVKey // TODO(b/162229294): This should be resolved during the course of that bug
 @Access(AccessType.FIELD)
-public class HistoryEntry extends ImmutableObject implements Buildable {
+public class HistoryEntry extends ImmutableObject implements Buildable, DatastoreEntity {
 
   /** Represents the type of history entry. */
   public enum Type {
@@ -175,7 +178,7 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
   boolean bySuperuser;
 
   /** Reason for the change. */
-  @Column(nullable = false, name = "historyReason")
+  @Column(name = "historyReason")
   String reason;
 
   /** Whether this change was requested by a registrar. */
@@ -199,7 +202,7 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
     return id == null ? 0L : id;
   }
 
-  // This method is required by Hibernate.
+  /** This method exists solely to satisfy Hibernate. Use the {@link Builder} instead. */
   @SuppressWarnings("UnusedMethod")
   private void setId(long id) {
     this.id = id;
@@ -251,10 +254,28 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
     return requestedByRegistrar;
   }
 
-  public ImmutableSet<DomainTransactionRecord> getDomainTransactionRecords() {
+  public Set<DomainTransactionRecord> getDomainTransactionRecords() {
     return nullToEmptyImmutableCopy(domainTransactionRecords);
   }
 
+  /** This method exists solely to satisfy Hibernate. Use the {@link Builder} instead. */
+  @SuppressWarnings("UnusedMethod")
+  private void setPeriod(Period period) {
+    this.period = period;
+  }
+
+  /** This method exists solely to satisfy Hibernate. Use the {@link Builder} instead. */
+  @SuppressWarnings("UnusedMethod")
+  private void setOtherRegistrarId(String otherRegistrarId) {
+    this.otherClientId = otherRegistrarId;
+  }
+
+  /** This method exists solely to satisfy Hibernate. Use the {@link Builder} instead. */
+  @SuppressWarnings("UnusedMethod")
+  private void setDomainTransactionRecords(Set<DomainTransactionRecord> domainTransactionRecords) {
+    this.domainTransactionRecords = ImmutableSet.copyOf(domainTransactionRecords);
+  }
+
   public static VKey<HistoryEntry> createVKey(Key<HistoryEntry> key) {
     // TODO(b/159207551): This will likely need some revision.  As it stands, this method was
     // introduced purely to facilitate testing of VKey specialization in VKeyTranslatorFactory.
@@ -287,19 +308,10 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
           new DomainHistory.Builder().copyFrom(this).setDomainRepoId(parent.getName()).build();
     } else if (parentKind.equals(getKind(HostResource.class))) {
       resultEntity =
-          new HostHistory.Builder()
-              .copyFrom(this)
-              .setHostRepoId(
-                  VKey.create(HostResource.class, parent.getName(), (Key<HostResource>) parent))
-              .build();
+          new HostHistory.Builder().copyFrom(this).setHostRepoId(parent.getName()).build();
     } else if (parentKind.equals(getKind(ContactResource.class))) {
       resultEntity =
-          new ContactHistory.Builder()
-              .copyFrom(this)
-              .setContactRepoId(
-                  VKey.create(
-                      ContactResource.class, parent.getName(), (Key<ContactResource>) parent))
-              .build();
+          new ContactHistory.Builder().copyFrom(this).setContactRepoId(parent.getName()).build();
     } else {
       throw new IllegalStateException(
           String.format("Unknown kind of HistoryEntry parent %s", parentKind));
@@ -307,6 +319,12 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
     return resultEntity;
   }
 
+  // In SQL, save the child type
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of((SqlEntity) toChildHistoryEntity());
+  }
+
   /** A builder for {@link HistoryEntry} since it is immutable */
   public static class Builder<T extends HistoryEntry, B extends Builder<?, ?>>
       extends GenericBuilder<T, B> {
@@ -331,7 +349,10 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
       setBySuperuser(historyEntry.bySuperuser);
       setReason(historyEntry.reason);
       setRequestedByRegistrar(historyEntry.requestedByRegistrar);
-      setDomainTransactionRecords(nullToEmptyImmutableCopy(historyEntry.domainTransactionRecords));
+      setDomainTransactionRecords(
+          historyEntry.domainTransactionRecords == null
+              ? null
+              : ImmutableSet.copyOf(historyEntry.domainTransactionRecords));
       return thisCastToDerived();
     }
 

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -46,6 +46,7 @@ import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.annotations.VirtualEntity;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import google.registry.util.CollectionUtils;
@@ -97,7 +98,7 @@ import org.joda.time.DateTime;
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 @javax.persistence.Entity(name = "ClaimsList")
 @Table
-public class ClaimsListShard extends ImmutableObject implements DatastoreEntity {
+public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlEntity {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -354,11 +355,6 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
     }
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // ClaimsLists are dually written
-  }
-
   /** Virtual parent entity for claims list shards of a specific revision. */
   @Entity
   @VirtualEntity

core/src/main/java/google/registry/model/transfer/DomainTransferData.java

@@ -14,12 +14,16 @@
 
 package google.registry.model.transfer;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.AlsoLoad;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Unindex;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.Period.Unit;
 import google.registry.model.poll.PollMessage;
@@ -86,6 +90,10 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
   @Column(name = "transfer_billing_event_id")
   VKey<BillingEvent.OneTime> serverApproveBillingEvent;
 
+  @Ignore
+  @Column(name = "transfer_billing_event_history_id")
+  Long serverApproveBillingEventHistoryId;
+
   /**
    * The autorenew billing event that should be associated with this resource after the transfer.
    *
@@ -96,6 +104,10 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
   @Column(name = "transfer_billing_recurrence_id")
   VKey<BillingEvent.Recurring> serverApproveAutorenewEvent;
 
+  @Ignore
+  @Column(name = "transfer_billing_recurrence_history_id")
+  Long serverApproveAutorenewEventHistoryId;
+
   /**
    * The autorenew poll message that should be associated with this resource after the transfer.
    *
@@ -106,11 +118,47 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
   @Column(name = "transfer_autorenew_poll_message_id")
   VKey<PollMessage.Autorenew> serverApproveAutorenewPollMessage;
 
+  @Ignore
+  @Column(name = "transfer_autorenew_poll_message_history_id")
+  Long serverApproveAutorenewPollMessageHistoryId;
+
   @Override
   public Builder copyConstantFieldsToBuilder() {
     return super.copyConstantFieldsToBuilder().setTransferPeriod(this.transferPeriod);
   }
 
+  /**
+   * Restores the set of ofy keys after loading from SQL using the specified {@code rootKey}.
+   *
+   * <p>This is for use by DomainBase/DomainHistory PostLoad methods ONLY.
+   */
+  public void restoreOfyKeys(Key<DomainBase> rootKey) {
+    serverApproveBillingEvent =
+        DomainBase.restoreOfyFrom(
+            rootKey, serverApproveBillingEvent, serverApproveBillingEventHistoryId);
+    serverApproveAutorenewEvent =
+        DomainBase.restoreOfyFrom(
+            rootKey, serverApproveAutorenewEvent, serverApproveAutorenewEventHistoryId);
+    serverApproveAutorenewPollMessage =
+        DomainBase.restoreOfyFrom(
+            rootKey, serverApproveAutorenewPollMessage, serverApproveAutorenewPollMessageHistoryId);
+  }
+
+  private void loadServerApproveBillingEventHistoryId(
+      @AlsoLoad("serverApproveBillingEvent") VKey<BillingEvent.OneTime> val) {
+    serverApproveBillingEventHistoryId = DomainBase.getHistoryId(val);
+  }
+
+  private void loadServerApproveAutorenewEventHistoryId(
+      @AlsoLoad("serverApproveAutorenewEvent") VKey<BillingEvent.Recurring> val) {
+    serverApproveAutorenewEventHistoryId = DomainBase.getHistoryId(val);
+  }
+
+  private void loadServerApproveAutorenewPollMessageHistoryId(
+      @AlsoLoad("serverApproveAutorenewPollMessage") VKey<PollMessage.Autorenew> val) {
+    serverApproveAutorenewPollMessageHistoryId = DomainBase.getHistoryId(val);
+  }
+
   public Period getTransferPeriod() {
     return transferPeriod;
   }
@@ -125,16 +173,34 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     return serverApproveBillingEvent;
   }
 
+  @VisibleForTesting
+  @Nullable
+  public Long getServerApproveBillingEventHistoryId() {
+    return serverApproveBillingEventHistoryId;
+  }
+
   @Nullable
   public VKey<BillingEvent.Recurring> getServerApproveAutorenewEvent() {
     return serverApproveAutorenewEvent;
   }
 
+  @VisibleForTesting
+  @Nullable
+  public Long getServerApproveAutorenewEventHistoryId() {
+    return serverApproveAutorenewEventHistoryId;
+  }
+
   @Nullable
   public VKey<PollMessage.Autorenew> getServerApproveAutorenewPollMessage() {
     return serverApproveAutorenewPollMessage;
   }
 
+  @VisibleForTesting
+  @Nullable
+  public Long getServerApproveAutorenewPollMessageHistoryId() {
+    return serverApproveAutorenewPollMessageHistoryId;
+  }
+
   @Override
   public boolean isEmpty() {
     return EMPTY.equals(this);
@@ -168,18 +234,24 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     public Builder setServerApproveBillingEvent(
         VKey<BillingEvent.OneTime> serverApproveBillingEvent) {
       getInstance().serverApproveBillingEvent = serverApproveBillingEvent;
+      getInstance().serverApproveBillingEventHistoryId =
+          DomainBase.getHistoryId(serverApproveBillingEvent);
       return this;
     }
 
     public Builder setServerApproveAutorenewEvent(
         VKey<BillingEvent.Recurring> serverApproveAutorenewEvent) {
       getInstance().serverApproveAutorenewEvent = serverApproveAutorenewEvent;
+      getInstance().serverApproveAutorenewEventHistoryId =
+          DomainBase.getHistoryId(serverApproveAutorenewEvent);
       return this;
     }
 
     public Builder setServerApproveAutorenewPollMessage(
         VKey<PollMessage.Autorenew> serverApproveAutorenewPollMessage) {
       getInstance().serverApproveAutorenewPollMessage = serverApproveAutorenewPollMessage;
+      getInstance().serverApproveAutorenewPollMessageHistoryId =
+          DomainBase.getHistoryId(serverApproveAutorenewPollMessage);
       return this;
     }
   }

core/src/main/java/google/registry/persistence/VKey.java

@@ -88,7 +88,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    */
   public static <T> VKey<T> create(Class<T> kind, long id) {
     checkArgument(
-        kind.isAssignableFrom(BackupGroupRoot.class),
+        BackupGroupRoot.class.isAssignableFrom(kind),
         "The kind %s is not a BackupGroupRoot and thus needs its entire entity group chain"
             + " specified in a parent",
         kind.getCanonicalName());
@@ -106,13 +106,90 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    */
   public static <T> VKey<T> create(Class<T> kind, String name) {
     checkArgument(
-        kind.isAssignableFrom(BackupGroupRoot.class),
+        BackupGroupRoot.class.isAssignableFrom(kind),
         "The kind %s is not a BackupGroupRoot and thus needs its entire entity group chain"
             + " specified in a parent",
         kind.getCanonicalName());
     return new VKey<T>(kind, Key.create(kind, name), name);
   }
 
+  /**
+   * Returns a clone with an ofy key restored from {@code ancestors}.
+   *
+   * <p>The arguments should generally consist of pairs of Class and value, where the Class is the
+   * kind of the ancestor key and the value is either a String or a Long.
+   *
+   * <p>For example, to restore the objectify key for
+   * DomainBase("COM-1234")/HistoryEntry(123)/PollEvent(567), one might use:
+   *
+   * <pre>{@code
+   * pollEvent.restoreOfy(DomainBase.class, "COM-1234", HistoryEntry.class, 567)
+   * }</pre>
+   *
+   * <p>The final key id or name is obtained from the SQL key. It is assumed that this value must be
+   * either a long integer or a {@code String} and that this proper identifier for the objectify
+   * key.
+   *
+   * <p>As a special case, an objectify Key may be used as the first ancestor instead of a Class,
+   * value pair.
+   */
+  public VKey<T> restoreOfy(Object... ancestors) {
+    Class lastClass = null;
+    Key<?> lastKey = null;
+    for (Object ancestor : ancestors) {
+      if (ancestor instanceof Class) {
+        if (lastClass != null) {
+          throw new IllegalArgumentException(ancestor + " used as a key value.");
+        }
+        lastClass = (Class) ancestor;
+        continue;
+      } else if (ancestor instanceof Key) {
+        if (lastKey != null) {
+          throw new IllegalArgumentException(
+              "Objectify keys may only be used for the first argument");
+        }
+        lastKey = (Key) ancestor;
+        continue;
+      }
+
+      // The argument should be a value.
+      if (lastClass == null) {
+        throw new IllegalArgumentException("Argument " + ancestor + " should be a class.");
+      }
+      if (ancestor instanceof Long) {
+        lastKey = Key.create(lastKey, lastClass, (Long) ancestor);
+      } else if (ancestor instanceof String) {
+        lastKey = Key.create(lastKey, lastClass, (String) ancestor);
+      } else {
+        throw new IllegalArgumentException("Key value " + ancestor + " must be a string or long.");
+      }
+      lastClass = null;
+    }
+
+    // Make sure we didn't end up with a dangling class with no value.
+    if (lastClass != null) {
+      throw new IllegalArgumentException("Missing value for last key of type " + lastClass);
+    }
+
+    Object sqlKey = getSqlKey();
+    Key<T> ofyKey =
+        sqlKey instanceof Long
+            ? Key.create(lastKey, getKind(), (Long) sqlKey)
+            : Key.create(lastKey, getKind(), (String) sqlKey);
+
+    return VKey.create((Class<T>) getKind(), sqlKey, ofyKey);
+  }
+
+  /**
+   * Returns a clone of {@code key} with an ofy key restored from {@code ancestors}.
+   *
+   * <p>This is the static form of the method restoreOfy() above. If {@code key} is null, it returns
+   * null.
+   */
+  public static <T> VKey<T> restoreOfyFrom(@Nullable VKey<T> key, Object... ancestors) {
+    return key == null ? null : key.restoreOfy(ancestors);
+  }
+
   /** Returns the type of the entity. */
   public Class<? extends T> getKind() {
     return this.kind;

core/src/main/java/google/registry/persistence/converter/CreateAutoTimestampConverter.java

@@ -32,7 +32,11 @@ public class CreateAutoTimestampConverter
     implements AttributeConverter<CreateAutoTimestamp, Timestamp> {
 
   @Override
-  public Timestamp convertToDatabaseColumn(CreateAutoTimestamp entity) {
+  @Nullable
+  public Timestamp convertToDatabaseColumn(@Nullable CreateAutoTimestamp entity) {
+    if (entity == null) {
+      return null;
+    }
     DateTime dateTime = firstNonNull(entity.getTimestamp(), jpaTm().getTransactionTime());
     return Timestamp.from(DateTimeUtils.toZonedDateTime(dateTime).toInstant());
   }

core/src/main/java/google/registry/persistence/converter/PremiumListKeyConverter.java

@@ -0,0 +1,37 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.registry.label.PremiumList;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA converter for a {@link Key} containing a {@link PremiumList} */
+@Converter(autoApply = true)
+public class PremiumListKeyConverter implements AttributeConverter<Key<PremiumList>, String> {
+
+  @Override
+  public String convertToDatabaseColumn(Key<PremiumList> attribute) {
+    return (attribute == null) ? null : attribute.getName();
+  }
+
+  @Override
+  public Key<PremiumList> convertToEntityAttribute(String dbData) {
+    return (dbData == null) ? null : Key.create(getCrossTldKey(), PremiumList.class, dbData);
+  }
+}

core/src/main/java/google/registry/persistence/converter/ReservedListKeySetConverter.java

@@ -0,0 +1,36 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.registry.label.ReservedList;
+import javax.persistence.Converter;
+
+/** JPA converter for a set of {@link Key} containing a {@link ReservedList} */
+@Converter(autoApply = true)
+public class ReservedListKeySetConverter extends StringSetConverterBase<Key<ReservedList>> {
+
+  @Override
+  String toString(Key<ReservedList> key) {
+    return key.getName();
+  }
+
+  @Override
+  Key<ReservedList> fromString(String value) {
+    return Key.create(getCrossTldKey(), ReservedList.class, value);
+  }
+}

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -224,7 +224,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void saveNew(Object entity) {
+  public void insert(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
     assertInTransaction();
     getEntityManager().persist(entity);
@@ -232,14 +232,14 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void saveAllNew(ImmutableCollection<?> entities) {
+  public void insertAll(ImmutableCollection<?> entities) {
     checkArgumentNotNull(entities, "entities must be specified");
     assertInTransaction();
-    entities.forEach(this::saveNew);
+    entities.forEach(this::insert);
   }
 
   @Override
-  public void saveNewOrUpdate(Object entity) {
+  public void put(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
     assertInTransaction();
     getEntityManager().merge(entity);
@@ -247,17 +247,17 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void saveNewOrUpdateAll(ImmutableCollection<?> entities) {
+  public void putAll(ImmutableCollection<?> entities) {
     checkArgumentNotNull(entities, "entities must be specified");
     assertInTransaction();
-    entities.forEach(this::saveNewOrUpdate);
+    entities.forEach(this::put);
   }
 
   @Override
   public void update(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
     assertInTransaction();
-    checkArgument(checkExists(entity), "Given entity does not exist");
+    checkArgument(exists(entity), "Given entity does not exist");
     getEntityManager().merge(entity);
     transactionInfo.get().addUpdate(entity);
   }
@@ -270,22 +270,22 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public <T> boolean checkExists(VKey<T> key) {
+  public <T> boolean exists(VKey<T> key) {
     checkArgumentNotNull(key, "key must be specified");
     EntityType<?> entityType = getEntityType(key.getKind());
     ImmutableSet<EntityId> entityIds = getEntityIdsFromSqlKey(entityType, key.getSqlKey());
-    return checkExists(entityType.getName(), entityIds);
+    return exists(entityType.getName(), entityIds);
   }
 
   @Override
-  public boolean checkExists(Object entity) {
+  public boolean exists(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
     EntityType<?> entityType = getEntityType(entity.getClass());
     ImmutableSet<EntityId> entityIds = getEntityIdsFromEntity(entityType, entity);
-    return checkExists(entityType.getName(), entityIds);
+    return exists(entityType.getName(), entityIds);
   }
 
-  private boolean checkExists(String entityName, ImmutableSet<EntityId> entityIds) {
+  private boolean exists(String entityName, ImmutableSet<EntityId> entityIds) {
     assertInTransaction();
     TypedQuery<Integer> query =
         getEntityManager()
@@ -391,7 +391,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   private static ImmutableSet<EntityId> getEntityIdsFromEntity(
       EntityType<?> entityType, Object entity) {
     if (entityType.hasSingleIdAttribute()) {
-      String idName = entityType.getDeclaredId(entityType.getIdType().getJavaType()).getName();
+      String idName = entityType.getId(entityType.getIdType().getJavaType()).getName();
       Object idValue = getFieldValue(entity, idName);
       return ImmutableSet.of(new EntityId(idName, idValue));
     } else {
@@ -402,7 +402,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   private static ImmutableSet<EntityId> getEntityIdsFromSqlKey(
       EntityType<?> entityType, Object sqlKey) {
     if (entityType.hasSingleIdAttribute()) {
-      String idName = entityType.getDeclaredId(entityType.getIdType().getJavaType()).getName();
+      String idName = entityType.getId(entityType.getIdType().getJavaType()).getName();
       return ImmutableSet.of(new EntityId(idName, sqlKey));
     } else {
       return getEntityIdsFromIdContainer(entityType, sqlKey);
@@ -429,7 +429,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
 
   private static Object getFieldValue(Object object, String fieldName) {
     try {
-      Field field = object.getClass().getDeclaredField(fieldName);
+      Field field = getField(object.getClass(), fieldName);
       field.setAccessible(true);
       return field.get(object);
     } catch (NoSuchFieldException | IllegalAccessException e) {
@@ -437,6 +437,21 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     }
   }
 
+  /** Gets the field definition from clazz or any superclass. */
+  private static Field getField(Class clazz, String fieldName) throws NoSuchFieldException {
+    try {
+      // Note that we have to use getDeclaredField() for this, getField() just finds public fields.
+      return clazz.getDeclaredField(fieldName);
+    } catch (NoSuchFieldException e) {
+      Class base = clazz.getSuperclass();
+      if (base != null) {
+        return getField(base, fieldName);
+      } else {
+        throw e;
+      }
+    }
+  }
+
   private static class TransactionInfo {
     EntityManager entityManager;
     boolean inTransaction = false;

core/src/main/java/google/registry/persistence/transaction/Transaction.java

@@ -208,7 +208,7 @@ public class Transaction extends ImmutableObject implements Buildable {
 
     @Override
     public void writeToDatastore() {
-      ofyTm().saveNewOrUpdate(entity);
+      ofyTm().put(entity);
     }
 
     @Override

core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java

@@ -14,6 +14,9 @@
 
 package google.registry.persistence.transaction;
 
+import com.google.common.collect.ImmutableList;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
@@ -27,7 +30,7 @@ import javax.persistence.Table;
  */
 @Entity
 @Table(name = "Transaction")
-public class TransactionEntity {
+public class TransactionEntity implements SqlEntity {
 
   @Id
   @GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -40,4 +43,9 @@ public class TransactionEntity {
   TransactionEntity(byte[] contents) {
     this.contents = contents;
   }
+
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(); // not stored in Datastore per se
+  }
 }

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -86,16 +86,16 @@ public interface TransactionManager {
   DateTime getTransactionTime();
 
   /** Persists a new entity in the database, throws exception if the entity already exists. */
-  void saveNew(Object entity);
+  void insert(Object entity);
 
   /** Persists all new entities in the database, throws exception if any entity already exists. */
-  void saveAllNew(ImmutableCollection<?> entities);
+  void insertAll(ImmutableCollection<?> entities);
 
   /** Persists a new entity or update the existing entity in the database. */
-  void saveNewOrUpdate(Object entity);
+  void put(Object entity);
 
   /** Persists all new entities or update the existing entities in the database. */
-  void saveNewOrUpdateAll(ImmutableCollection<?> entities);
+  void putAll(ImmutableCollection<?> entities);
 
   /** Updates an entity in the database, throws exception if the entity does not exist. */
   void update(Object entity);
@@ -104,10 +104,10 @@ public interface TransactionManager {
   void updateAll(ImmutableCollection<?> entities);
 
   /** Returns whether the given entity with same ID exists. */
-  boolean checkExists(Object entity);
+  boolean exists(Object entity);
 
   /** Returns whether the entity of given key exists. */
-  <T> boolean checkExists(VKey<T> key);
+  <T> boolean exists(VKey<T> key);
 
   /** Loads the entity by its id, returns empty if the entity doesn't exist. */
   <T> Optional<T> maybeLoad(VKey<T> key);

core/src/main/java/google/registry/reporting/icann/IcannHttpReporter.java

@@ -116,14 +116,10 @@ public class IcannHttpReporter {
       // 1000 (i. e. success), there is no need to parse it.
       if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_BAD_REQUEST) {
         success = false;
-        // To debug if there is a problem with our parsing, we wrap the response and print the stack
-        // trace of it. As far as we can tell, the stack trace for such an exception contains the
-        // response content that is decoded correctly using the expected charset.
-        new HttpResponseException(response).printStackTrace();
         XjcIirdeaResult result = parseResult(content);
         logger.atWarning().log(
             "PUT rejected, status code %s:\n%s\n%s",
-            result.getCode(), result.getMsg(), result.getDescription());
+            result.getCode().getValue(), result.getMsg(), result.getDescription());
       }
     } finally {
       if (response != null) {

core/src/main/java/google/registry/tools/CreateOrUpdateTldCommand.java

@@ -110,6 +110,12 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
       description = "One-time billing cost for a server status change")
   private Money serverStatusChangeCost;
 
+  @Nullable
+  @Parameter(
+      names = "--registry_lock_or_unlock_cost",
+      description = "One-time billing cost for a registry lock or unlock")
+  private Money registryLockOrUnlockCost;
+
   @Nullable
   @Parameter(
       names = "--tld_type",
@@ -118,10 +124,10 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
 
   @Nullable
   @Parameter(
-      names = "--disable_invoicing",
-      description = "Whether invoicing is disabled for a REAL tld.",
+      names = "--invoicing_enabled",
+      description = "Whether invoicing is enabled for this tld.",
       arity = 1)
-  private Boolean disableInvoicing;
+  private Boolean invoicingEnabled;
 
   @Nullable
   @Parameter(
@@ -326,8 +332,10 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
       Optional.ofNullable(roidSuffix).ifPresent(builder::setRoidSuffix);
       Optional.ofNullable(serverStatusChangeCost)
           .ifPresent(builder::setServerStatusChangeBillingCost);
+      Optional.ofNullable(registryLockOrUnlockCost)
+          .ifPresent(builder::setRegistryLockOrUnlockBillingCost);
       Optional.ofNullable(tldType).ifPresent(builder::setTldType);
-      Optional.ofNullable(disableInvoicing).ifPresent(builder::setDisableInvoicing);
+      Optional.ofNullable(invoicingEnabled).ifPresent(builder::setInvoicingEnabled);
       Optional.ofNullable(lordnUsername).ifPresent(u -> builder.setLordnUsername(u.orElse(null)));
       Optional.ofNullable(claimsPeriodEnd).ifPresent(builder::setClaimsPeriodEnd);
       Optional.ofNullable(numDnsPublishShards).ifPresent(builder::setNumDnsPublishLocks);

core/src/main/java/google/registry/tools/CreateRegistrarCommand.java

@@ -72,7 +72,7 @@ final class CreateRegistrarCommand extends CreateOrUpdateRegistrarCommand
 
   @Override
   void saveToCloudSql(Registrar registrar) {
-    jpaTm().saveNew(registrar);
+    jpaTm().insert(registrar);
   }
 
   @Nullable

core/src/main/java/google/registry/tools/DomainLockUtils.java

@@ -380,7 +380,7 @@ public final class DomainLockUtils {
               .setReason(Reason.SERVER_STATUS)
               .setTargetId(domain.getForeignKey())
               .setClientId(domain.getCurrentSponsorClientId())
-              .setCost(Registry.get(domain.getTld()).getServerStatusChangeCost())
+              .setCost(Registry.get(domain.getTld()).getRegistryLockOrUnlockBillingCost())
               .setEventTime(now)
               .setBillingTime(now)
               .setParent(historyEntry)

core/src/main/resources/META-INF/orm.xml

@@ -11,8 +11,6 @@
     </attributes>
   </embeddable>
 
-  <sequence-generator name="HistorySequenceGenerator" sequence-name="history_id_sequence" />
-
   <persistence-unit-metadata>
     <persistence-unit-defaults>
       <entity-listeners>

core/src/main/resources/META-INF/persistence.xml

@@ -45,25 +45,27 @@
     <class>google.registry.model.contact.ContactResource</class>
     <class>google.registry.model.domain.DomainBase</class>
     <class>google.registry.model.domain.DomainHistory</class>
+    <class>google.registry.model.domain.GracePeriod</class>
+    <class>google.registry.model.domain.secdns.DelegationSignerData</class>
     <class>google.registry.model.domain.token.AllocationToken</class>
     <class>google.registry.model.host.HostHistory</class>
     <class>google.registry.model.host.HostResource</class>
-    <class>google.registry.model.registrar.Registrar</class>
-    <class>google.registry.model.registrar.RegistrarContact</class>
-    <class>google.registry.model.registry.label.PremiumList</class>
-    <class>google.registry.model.reporting.Spec11ThreatMatch</class>
-    <class>google.registry.persistence.transaction.TransactionEntity</class>
-    <class>google.registry.model.tmch.ClaimsListShard</class>
-    <class>google.registry.schema.domain.RegistryLock</class>
-    <class>google.registry.schema.cursor.Cursor</class>
-    <class>google.registry.schema.server.Lock</class>
-    <class>google.registry.schema.tld.PremiumEntry</class>
-    <class>google.registry.model.domain.secdns.DelegationSignerData</class>
-    <class>google.registry.model.domain.GracePeriod</class>
     <class>google.registry.model.poll.PollMessage</class>
     <class>google.registry.model.poll.PollMessage$OneTime</class>
     <class>google.registry.model.poll.PollMessage$Autorenew</class>
+    <class>google.registry.model.registrar.Registrar</class>
+    <class>google.registry.model.registrar.RegistrarContact</class>
+    <class>google.registry.model.registry.label.PremiumList</class>
     <class>google.registry.model.registry.label.ReservedList</class>
+    <class>google.registry.model.registry.Registry</class>
+    <class>google.registry.model.reporting.DomainTransactionRecord</class>
+    <class>google.registry.model.reporting.Spec11ThreatMatch</class>
+    <class>google.registry.model.tmch.ClaimsListShard</class>
+    <class>google.registry.persistence.transaction.TransactionEntity</class>
+    <class>google.registry.schema.cursor.Cursor</class>
+    <class>google.registry.schema.domain.RegistryLock</class>
+    <class>google.registry.schema.server.Lock</class>
+    <class>google.registry.schema.tld.PremiumEntry</class>
 
     <!-- Customized type converters -->
     <class>google.registry.persistence.converter.AllocationTokenStatusTransitionConverter</class>
@@ -79,7 +81,9 @@
     <class>google.registry.persistence.converter.InetAddressSetConverter</class>
     <class>google.registry.persistence.converter.LocalDateConverter</class>
     <class>google.registry.persistence.converter.PostalInfoChoiceListConverter</class>
+    <class>google.registry.persistence.converter.PremiumListKeyConverter</class>
     <class>google.registry.persistence.converter.RegistrarPocSetConverter</class>
+    <class>google.registry.persistence.converter.ReservedListKeySetConverter</class>
     <class>google.registry.persistence.converter.Spec11ThreatMatchThreatTypeSetConverter</class>
     <class>google.registry.persistence.converter.StatusValueSetConverter</class>
     <class>google.registry.persistence.converter.StringListConverter</class>
@@ -90,7 +94,10 @@
     <class>google.registry.persistence.converter.ZonedDateTimeConverter</class>
 
     <!-- Generated converters for VKey -->
-    <class>google.registry.model.billing.VKeyConverter_BillingEvent</class>
+    <class>google.registry.model.billing.VKeyConverter_Cancellation</class>
+    <class>google.registry.model.billing.VKeyConverter_Modification</class>
+    <class>google.registry.model.billing.VKeyConverter_OneTime</class>
+    <class>google.registry.model.billing.VKeyConverter_Recurring</class>
     <class>google.registry.model.contact.VKeyConverter_ContactResource</class>
     <class>google.registry.model.domain.VKeyConverter_DomainBase</class>
     <class>google.registry.model.domain.token.VKeyConverter_AllocationToken</class>

core/src/nonprod/java/google/registry/tools/DevTool.java

@@ -28,6 +28,7 @@ public class DevTool {
   public static final ImmutableMap<String, Class<? extends Command>> COMMAND_MAP =
       ImmutableMap.of(
           "dump_golden_schema", DumpGoldenSchemaCommand.class,
+          "generate_sql_er_diagram", GenerateSqlErDiagramCommand.class,
           "generate_sql_schema", GenerateSqlSchemaCommand.class);
 
   public static void main(String[] args) throws Exception {

core/src/nonprod/java/google/registry/tools/GenerateSqlErDiagramCommand.java

@@ -0,0 +1,227 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.tools.GenerateSqlErDiagramCommand.DiagramType.ALL;
+import static google.registry.tools.GenerateSqlErDiagramCommand.DiagramType.BRIEF;
+import static google.registry.tools.GenerateSqlErDiagramCommand.DiagramType.FULL;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.converters.PathConverter;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterables;
+import com.google.common.io.Resources;
+import google.registry.persistence.NomulusPostgreSql;
+import google.registry.util.ResourceUtils;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.testcontainers.containers.PostgreSQLContainer;
+import schemacrawler.schemacrawler.LoadOptionsBuilder;
+import schemacrawler.schemacrawler.SchemaCrawlerOptions;
+import schemacrawler.schemacrawler.SchemaCrawlerOptionsBuilder;
+import schemacrawler.schemacrawler.SchemaInfoLevelBuilder;
+import schemacrawler.tools.executable.SchemaCrawlerExecutable;
+import schemacrawler.tools.integration.diagram.DiagramOutputFormat;
+import schemacrawler.tools.options.OutputOptions;
+import schemacrawler.tools.options.OutputOptionsBuilder;
+
+/** Command to generate ER diagrams for SQL schema. */
+@Parameters(separators = " =", commandDescription = "Generate ER diagrams for SQL schmea.")
+public class GenerateSqlErDiagramCommand implements Command {
+
+  private static final String DB_NAME = "postgres";
+  private static final String DB_USER = "username";
+  private static final String DB_PASSWORD = "password";
+  private static final String FULL_DIAGRAM_COMMAND = "schema";
+  private static final String BRIEF_DIAGRAM_COMMAND = "brief";
+  private static final String FULL_DIAGRAM_FILE_NAME = "full_er_diagram.html";
+  private static final String BRIEF_DIAGRAM_FILE_NAME = "brief_er_diagram.html";
+  private static final String NOMULUS_GOLDEN_SCHEMA = "sql/schema/nomulus.golden.sql";
+  private static final String FLYWAY_FILE = "sql/flyway.txt";
+  private static final String SVG_PAN_ZOOM_LIB = "google/registry/tools/svg-pan-zoom.min.js";
+
+  // The HTML element ID for the last flyway file name
+  static final String FLYWAY_FILE_ELEMENT_ID = "lastFlywayFile";
+
+  @Parameter(
+      names = {"-o", "--out_dir"},
+      description = "Name of the output directory to store ER diagrams.",
+      converter = PathConverter.class,
+      required = true)
+  private Path outDir;
+
+  @Parameter(
+      names = "--diagram_type",
+      description =
+          "Type of the generated ER diagram, can be FULL, BRIEF and ALL (defaults to ALL).")
+  private DiagramType diagramType = ALL;
+
+  /** The type of ER diagram. */
+  public enum DiagramType {
+    /** An HTML file that has an embedded ER diagram showing the full SQL schema. */
+    FULL,
+
+    /**
+     * An HTML file that has an embedded ER diagram showing only significant columns, such as
+     * primary and foreign key columns, and columns that are part of unique indexes.
+     */
+    BRIEF,
+
+    /** Generates all types of ER diagrams. */
+    ALL
+  }
+
+  @Override
+  public void run() throws Exception {
+    if (!outDir.toFile().exists()) {
+      checkState(outDir.toFile().mkdirs(), "Failed to create directory %s", outDir);
+    }
+
+    PostgreSQLContainer postgresContainer =
+        new PostgreSQLContainer(NomulusPostgreSql.getDockerTag())
+            .withDatabaseName(DB_NAME)
+            .withUsername(DB_USER)
+            .withPassword(DB_PASSWORD);
+    postgresContainer.start();
+
+    try (Connection conn = getConnection(postgresContainer)) {
+      initDb(conn);
+      if (diagramType == ALL || diagramType == FULL) {
+        improveDiagramHtml(generateErDiagram(conn, FULL_DIAGRAM_COMMAND, FULL_DIAGRAM_FILE_NAME));
+      }
+      if (diagramType == ALL || diagramType == BRIEF) {
+        improveDiagramHtml(generateErDiagram(conn, BRIEF_DIAGRAM_COMMAND, BRIEF_DIAGRAM_FILE_NAME));
+      }
+    } finally {
+      postgresContainer.stop();
+    }
+  }
+
+  private void improveDiagramHtml(Path diagram) {
+    try {
+      Document doc = Jsoup.parse(diagram.toFile(), StandardCharsets.UTF_8.name());
+
+      // Add the last name of the flyway file to the HTML so we can have a test to verify that if
+      // the generated diagram is up to date.
+      doc.select("body > table > tbody")
+          .first()
+          .append(
+              String.format(
+                  "<tr>"
+                      + "<td class=\"property_name\">last flyway file</td>"
+                      + "<td id=\""
+                      + FLYWAY_FILE_ELEMENT_ID
+                      + "\" class=\"property_value\">"
+                      + getLastFlywayFileName()
+                      + "</td>"
+                      + "</tr>"));
+
+      // Add pan and zoom support for the embedded SVG in the HTML.
+      StringBuilder svgPanZoomLib =
+          new StringBuilder("<script>")
+              .append(ResourceUtils.readResourceUtf8(Resources.getResource(SVG_PAN_ZOOM_LIB)))
+              .append("</script>");
+      doc.select("head").first().append(svgPanZoomLib.toString());
+      doc.select("svg")
+          .first()
+          .attributes()
+          .add("id", "erDiagram")
+          .add("style", "overflow: hidden; width: 100%; height: 800px");
+      doc.select("body")
+          .first()
+          .append(
+              "<script>"
+                  + "svgPanZoom('#erDiagram', {"
+                  + "  zoomEnabled: true,"
+                  + "  controlIconsEnabled: true,"
+                  + "  fit: true,"
+                  + "  center: true,"
+                  + "  minZoom: 0.1"
+                  + "});"
+                  + "</script>");
+
+      Files.write(
+          diagram, doc.outerHtml().getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE);
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  private Path generateErDiagram(Connection connection, String command, String fileName) {
+    Path outputFile = outDir.resolve(fileName);
+
+    LoadOptionsBuilder loadOptionsBuilder =
+        LoadOptionsBuilder.builder().withSchemaInfoLevel(SchemaInfoLevelBuilder.standard());
+    SchemaCrawlerOptions options =
+        SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions()
+            .withLoadOptions(loadOptionsBuilder.toOptions());
+    OutputOptions outputOptions =
+        OutputOptionsBuilder.newOutputOptions(DiagramOutputFormat.htmlx, outputFile);
+
+    SchemaCrawlerExecutable executable = new SchemaCrawlerExecutable(command);
+    executable.setSchemaCrawlerOptions(options);
+    executable.setOutputOptions(outputOptions);
+    executable.setConnection(connection);
+    try {
+      executable.execute();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+
+    return outputFile;
+  }
+
+  private static Connection getConnection(PostgreSQLContainer container) {
+    Properties info = new Properties();
+    info.put("user", container.getUsername());
+    info.put("password", container.getPassword());
+    try {
+      return container.getJdbcDriverInstance().connect(container.getJdbcUrl(), info);
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private static void initDb(Connection connection) {
+    try (Statement statement = connection.createStatement()) {
+      statement.execute(
+          ResourceUtils.readResourceUtf8(Resources.getResource(NOMULUS_GOLDEN_SCHEMA)));
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @VisibleForTesting
+  static String getLastFlywayFileName() {
+    try {
+      return Iterables.getLast(
+          Resources.readLines(Resources.getResource(FLYWAY_FILE), StandardCharsets.UTF_8));
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+}

core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java

@@ -273,8 +273,11 @@ class InitSqlPipelineTest {
             "revisions",
             "updateTimestamp",
             "autorenewBillingEvent",
+            "autorenewBillingEventHistoryId",
             "autorenewPollMessage",
+            "autorenewPollMessageHistoryId",
             "deletePollMessage",
+            "deletePollMessageHistoryId",
             "nsHosts",
             "transferData");
     assertThat(actual.getAdminContact().getSqlKey())

core/src/test/java/google/registry/beam/initsql/WriteToSqlTest.java

@@ -85,7 +85,7 @@ class WriteToSqlTest implements Serializable {
       // Required for contacts created below.
       Registrar ofyRegistrar = AppEngineExtension.makeRegistrar2();
       store.insertOrUpdate(ofyRegistrar);
-      jpaTm().transact(() -> jpaTm().saveNewOrUpdate(store.loadAsOfyEntity(ofyRegistrar)));
+      jpaTm().transact(() -> jpaTm().put(store.loadAsOfyEntity(ofyRegistrar)));
 
       ImmutableList.Builder<Entity> builder = new ImmutableList.Builder<>();
 

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -285,7 +285,7 @@ class Spec11PipelineTest {
             .build();
 
     verify(mockJpaTm).transact(any(Runnable.class));
-    verify(mockJpaTm).saveNew(expected);
+    verify(mockJpaTm).insert(expected);
     verifyNoMoreInteractions(mockJpaTm);
   }
 

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -821,10 +821,12 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     setEppInput("domain_create_dsdata_no_maxsiglife.xml");
     persistContactsAndHosts("tld"); // For some reason this sample uses "tld".
     doSuccessfulTest("tld");
+    DomainBase domain = reloadResourceByForeignKey();
     assertAboutDomains()
-        .that(reloadResourceByForeignKey())
+        .that(domain)
         .hasExactlyDsData(
-            DelegationSignerData.create(12345, 3, 1, base16().decode("49FD46E6C4B45C55D4AC")));
+            DelegationSignerData.create(12345, 3, 1, base16().decode("49FD46E6C4B45C55D4AC"))
+                .cloneWithDomainRepoId(domain.getRepoId()));
   }
 
   @Test

core/src/test/java/google/registry/flows/domain/DomainRenewFlowTest.java

@@ -597,6 +597,7 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
             .setRestoreBillingCost(Money.of(EUR, 11))
             .setRenewBillingCostTransitions(ImmutableSortedMap.of(START_OF_TIME, Money.of(EUR, 7)))
             .setEapFeeSchedule(ImmutableSortedMap.of(START_OF_TIME, Money.zero(EUR)))
+            .setRegistryLockOrUnlockBillingCost(Money.of(EUR, 20))
             .setServerStatusChangeBillingCost(Money.of(EUR, 19))
             .build());
     persistDomain();
@@ -615,6 +616,7 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
             .setRestoreBillingCost(Money.of(EUR, 11))
             .setRenewBillingCostTransitions(ImmutableSortedMap.of(START_OF_TIME, Money.of(EUR, 7)))
             .setEapFeeSchedule(ImmutableSortedMap.of(START_OF_TIME, Money.zero(EUR)))
+            .setRegistryLockOrUnlockBillingCost(Money.of(EUR, 20))
             .setServerStatusChangeBillingCost(Money.of(EUR, 19))
             .build());
     persistDomain();
@@ -633,6 +635,7 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
             .setRestoreBillingCost(Money.of(EUR, 11))
             .setRenewBillingCostTransitions(ImmutableSortedMap.of(START_OF_TIME, Money.of(EUR, 7)))
             .setEapFeeSchedule(ImmutableSortedMap.of(START_OF_TIME, Money.zero(EUR)))
+            .setRegistryLockOrUnlockBillingCost(Money.of(EUR, 20))
             .setServerStatusChangeBillingCost(Money.of(EUR, 19))
             .build());
     persistDomain();

core/src/test/java/google/registry/flows/domain/DomainRestoreRequestFlowTest.java

@@ -533,6 +533,7 @@ class DomainRestoreRequestFlowTest
             .setRenewBillingCostTransitions(ImmutableSortedMap.of(START_OF_TIME, Money.of(EUR, 7)))
             .setEapFeeSchedule(ImmutableSortedMap.of(START_OF_TIME, Money.zero(EUR)))
             .setServerStatusChangeBillingCost(Money.of(EUR, 19))
+            .setRegistryLockOrUnlockBillingCost(Money.of(EUR, 0))
             .build());
     EppException thrown = assertThrows(CurrencyUnitMismatchException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.flows.domain;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Sets.union;
 import static com.google.common.io.BaseEncoding.base16;
 import static com.google.common.truth.Truth.assertThat;
@@ -470,7 +471,11 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         .that(resource)
         .hasOnlyOneHistoryEntryWhich()
         .hasType(HistoryEntry.Type.DOMAIN_UPDATE);
-    assertThat(resource.getDsData()).isEqualTo(expectedDsData);
+    assertThat(resource.getDsData())
+        .isEqualTo(
+            expectedDsData.stream()
+                .map(ds -> ds.cloneWithDomainRepoId(resource.getRepoId()))
+                .collect(toImmutableSet()));
     assertDnsTasksEnqueued("example.tld");
   }