Fix accessing superclass fields in checkExists() (#799 )

* Fix accessing superclass fields in checkExists() JpaTransactionManagerImpl doesn't respect @Id fields in mapped superclasses. Replace calls to getDeclaredId() and getDeclaredField() with superclass friendly counterparts.
Add and use temp_history_id_sequence to avoid release error (#795 )
2026-07-09 01:27:09 +00:00 · 2020-09-11 13:45:51 -04:00 · 2020-09-11 12:25:08 -04:00 · 2020-09-10 17:36:22 -04:00 · 2020-09-10 12:02:24 -04:00 · 2020-09-09 19:37:41 -04:00
60 changed files with 937 additions and 413 deletions
@@ -6,9 +6,9 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.10.2
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.fasterxml.jackson.core:jackson-databind:2.10.2
+com.fasterxml.jackson.core:jackson-annotations:2.11.2
+com.fasterxml.jackson.core:jackson-core:2.11.2
+com.fasterxml.jackson.core:jackson-databind:2.11.2
 com.fasterxml:classmate:1.5.1
 com.github.jnr:jffi:1.2.23
 com.github.jnr:jnr-a64asm:1.0.0
@@ -6,9 +6,9 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.10.2
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.fasterxml.jackson.core:jackson-databind:2.10.2
+com.fasterxml.jackson.core:jackson-annotations:2.11.2
+com.fasterxml.jackson.core:jackson-core:2.11.2
+com.fasterxml.jackson.core:jackson-databind:2.11.2
 com.fasterxml:classmate:1.5.1
 com.github.jnr:jffi:1.2.23
 com.github.jnr:jnr-a64asm:1.0.0
@@ -6,9 +6,9 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.10.2
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.fasterxml.jackson.core:jackson-databind:2.10.2
+com.fasterxml.jackson.core:jackson-annotations:2.11.2
+com.fasterxml.jackson.core:jackson-core:2.11.2
+com.fasterxml.jackson.core:jackson-databind:2.11.2
 com.fasterxml:classmate:1.5.1
 com.github.jnr:jffi:1.2.23
 com.github.jnr:jnr-a64asm:1.0.0
@@ -6,9 +6,9 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.33
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.10.2
-com.fasterxml.jackson.core:jackson-core:2.10.2
-com.fasterxml.jackson.core:jackson-databind:2.10.2
+com.fasterxml.jackson.core:jackson-annotations:2.11.2
+com.fasterxml.jackson.core:jackson-core:2.11.2
+com.fasterxml.jackson.core:jackson-databind:2.11.2
 com.fasterxml:classmate:1.5.1
 com.github.jnr:jffi:1.2.23
 com.github.jnr:jnr-a64asm:1.0.0
@@ -55,9 +55,7 @@ FROM (
    FROM
      `%PROJECT_ID%.%DATASTORE_EXPORT_DATA_SET%.%REGISTRY_TABLE%`
    WHERE
-    -- TODO(b/18092292): Add a filter for tldState (not PDT/PREDELEGATION)
-      tldType = 'REAL'
-    AND disableInvoicing is not TRUE) ) AS BillingEvent
+      enableInvoicing IS TRUE) ) AS BillingEvent
  -- Gather billing ID from registrar table
  -- This is a 'JOIN' as opposed to 'LEFT JOIN' to filter out
  -- non-billable registrars
@@ -46,6 +46,7 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -274,7 +275,7 @@ public abstract class BillingEvent extends ImmutableObject
        @javax.persistence.Index(columnList = "allocation_token_id")
      })
  @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
-  public static class OneTime extends BillingEvent {
+  public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {

    /** The billable value. */
    @AttributeOverrides({
@@ -450,7 +451,7 @@ public abstract class BillingEvent extends ImmutableObject
        @javax.persistence.Index(columnList = "recurrence_time_of_year")
      })
  @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
-  public static class Recurring extends BillingEvent {
+  public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity {

    /**
     * The billing event recurs every year between {@link #eventTime} and this time on the
@@ -544,7 +545,7 @@ public abstract class BillingEvent extends ImmutableObject
        @javax.persistence.Index(columnList = "billingTime")
      })
  @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
-  public static class Cancellation extends BillingEvent {
+  public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity {

    /** The billing time of the charge that is being cancelled. */
    @Index
@@ -664,7 +665,7 @@ public abstract class BillingEvent extends ImmutableObject
  /** An event representing a modification of an existing one-time billing event. */
  @ReportedOn
  @Entity
-  public static class Modification extends BillingEvent {
+  public static class Modification extends BillingEvent implements DatastoreAndSqlEntity {

    /** The change in cost that should be applied to the original billing event. */
    Money cost;
@@ -19,8 +19,13 @@ import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;

 /**
 * A persisted history entry representing an EPP modification to a contact.
@@ -38,13 +43,24 @@ import javax.persistence.Entity;
      @javax.persistence.Index(columnList = "historyModificationTime")
    })
@EntitySubclass
+@Access(AccessType.FIELD)
 public class ContactHistory extends HistoryEntry {
+
  // Store ContactBase instead of ContactResource so we don't pick up its @Id
  ContactBase contactBase;

  @Column(nullable = false)
  VKey<ContactResource> contactRepoId;

+  @Id
+  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "TempHistorySequenceGenerator")
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
  /** The state of the {@link ContactBase} object at this point in time. */
  public ContactBase getContactBase() {
    return contactBase;
@@ -14,20 +14,32 @@

 package google.registry.model.domain;

+import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
+import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.EppResource;
-import google.registry.model.contact.ContactResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import java.io.Serializable;
 import java.util.Set;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Index;
 import javax.persistence.JoinTable;
+import javax.persistence.PostLoad;
+import javax.persistence.Table;

 /**
 * A persisted history entry representing an EPP modification to a domain.
@@ -37,27 +49,44 @@ import javax.persistence.JoinTable;
 * the foreign-keyed fields in that class can refer to this object.
 */
@Entity
-@javax.persistence.Table(
+@Table(
    indexes = {
-      @javax.persistence.Index(columnList = "creationTime"),
-      @javax.persistence.Index(columnList = "historyRegistrarId"),
-      @javax.persistence.Index(columnList = "historyType"),
-      @javax.persistence.Index(columnList = "historyModificationTime")
+      @Index(columnList = "creationTime"),
+      @Index(columnList = "historyRegistrarId"),
+      @Index(columnList = "historyType"),
+      @Index(columnList = "historyModificationTime")
    })
@EntitySubclass
+@Access(AccessType.FIELD)
+@IdClass(DomainHistoryId.class)
 public class DomainHistory extends HistoryEntry {
+
  // Store DomainContent instead of DomainBase so we don't pick up its @Id
  DomainContent domainContent;

-  @Column(nullable = false)
-  VKey<DomainBase> domainRepoId;
+  @Id String domainRepoId;

+  // We could have reused domainContent.nsHosts here, but Hibernate throws a weird exception after
+  // we change to use a composite primary key.
+  // TODO(b/166776754): Investigate if we can reuse domainContent.nsHosts for storing host keys.
+  @Ignore
  @ElementCollection
  @JoinTable(name = "DomainHistoryHost")
-  @Access(AccessType.PROPERTY)
  @Column(name = "host_repo_id")
+  Set<VKey<HostResource>> nsHosts;
+
+  @Id
+  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "TempHistorySequenceGenerator")
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
+  /** Returns keys to the {@link HostResource} that are the nameservers for the domain. */
  public Set<VKey<HostResource>> getNsHosts() {
-    return domainContent.nsHosts;
+    return nsHosts;
  }

  /** The state of the {@link DomainContent} object at this point in time. */
@@ -65,16 +94,51 @@ public class DomainHistory extends HistoryEntry {
    return domainContent;
  }

-  /** The key to the {@link ContactResource} this is based off of. */
+  /** The key to the {@link DomainBase} this is based off of. */
  public VKey<DomainBase> getDomainRepoId() {
-    return domainRepoId;
+    return VKey.create(DomainBase.class, domainRepoId, Key.create(DomainBase.class, domainRepoId));
  }

-  // Hibernate needs this in order to populate nsHosts but no one else should ever use it
-  @SuppressWarnings("UnusedMethod")
-  private void setNsHosts(Set<VKey<HostResource>> nsHosts) {
+  public VKey<DomainHistory> createVKey() {
+    return VKey.createSql(DomainHistory.class, new DomainHistoryId(domainRepoId, getId()));
+  }
+
+  @PostLoad
+  void postLoad() {
    if (domainContent != null) {
-      domainContent.nsHosts = nsHosts;
+      domainContent.nsHosts = nullToEmptyImmutableCopy(nsHosts);
+    }
+  }
+
+  /** Class to represent the composite primary key of {@link DomainHistory} entity. */
+  static class DomainHistoryId extends ImmutableObject implements Serializable {
+
+    private String domainRepoId;
+
+    private Long id;
+
+    /** Hibernate requires this default constructor. */
+    private DomainHistoryId() {}
+
+    DomainHistoryId(String domainRepoId, long id) {
+      this.domainRepoId = domainRepoId;
+      this.id = id;
+    }
+
+    String getDomainRepoId() {
+      return domainRepoId;
+    }
+
+    void setDomainRepoId(String domainRepoId) {
+      this.domainRepoId = domainRepoId;
+    }
+
+    long getId() {
+      return id;
+    }
+
+    void setId(long id) {
+      this.id = id;
    }
  }

@@ -93,12 +157,15 @@ public class DomainHistory extends HistoryEntry {

    public Builder setDomainContent(DomainContent domainContent) {
      getInstance().domainContent = domainContent;
+      if (domainContent != null) {
+        getInstance().nsHosts = nullToEmptyImmutableCopy(domainContent.nsHosts);
+      }
      return this;
    }

-    public Builder setDomainRepoId(VKey<DomainBase> domainRepoId) {
+    public Builder setDomainRepoId(String domainRepoId) {
      getInstance().domainRepoId = domainRepoId;
-      domainRepoId.maybeGetOfyKey().ifPresent(parent -> getInstance().parent = parent);
+      getInstance().parent = Key.create(DomainBase.class, domainRepoId);
      return this;
    }

@@ -106,8 +173,7 @@ public class DomainHistory extends HistoryEntry {
    @Override
    public Builder setParent(Key<? extends EppResource> parent) {
      super.setParent(parent);
-      getInstance().domainRepoId =
-          VKey.create(DomainBase.class, parent.getName(), (Key<DomainBase>) parent);
+      getInstance().domainRepoId = parent.getName();
      return this;
    }
  }
@@ -19,8 +19,13 @@ import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;

 /**
 * A persisted history entry representing an EPP modification to a host.
@@ -39,6 +44,7 @@ import javax.persistence.Entity;
      @javax.persistence.Index(columnList = "historyModificationTime")
    })
@EntitySubclass
+@Access(AccessType.FIELD)
 public class HostHistory extends HistoryEntry {

  // Store HostBase instead of HostResource so we don't pick up its @Id
@@ -47,6 +53,15 @@ public class HostHistory extends HistoryEntry {
  @Column(nullable = false)
  VKey<HostResource> hostRepoId;

+  @Id
+  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "TempHistorySequenceGenerator")
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
  /** The state of the {@link HostBase} object at this point in time. */
  public HostBase getHostBase() {
    return hostBase;
@@ -17,6 +17,7 @@ package google.registry.model.index;
 import static google.registry.util.TypeUtils.instantiate;

 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -25,11 +26,13 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.EppResource;
 import google.registry.model.annotations.ReportedOn;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;

 /** An index that allows for quick enumeration of all EppResource entities (e.g. via map reduce). */
@ReportedOn
@Entity
-public class EppResourceIndex extends BackupGroupRoot {
+public class EppResourceIndex extends BackupGroupRoot implements DatastoreEntity {

  @Id
  String id;
@@ -74,4 +77,9 @@ public class EppResourceIndex extends BackupGroupRoot {
  public static <T extends EppResource> EppResourceIndex create(Key<T> resourceKey) {
    return create(EppResourceIndexBucket.getBucketKey(resourceKey), resourceKey);
  }
+
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not relevant in SQL
+  }
 }
@@ -24,16 +24,23 @@ import com.googlecode.objectify.annotation.Id;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.VirtualEntity;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;

 /** A virtual entity to represent buckets to which EppResourceIndex objects are randomly added. */
@Entity
@VirtualEntity
-public class EppResourceIndexBucket extends ImmutableObject {
+public class EppResourceIndexBucket extends ImmutableObject implements DatastoreEntity {

  @SuppressWarnings("unused")
  @Id
  private long bucketId;

+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(); // not relevant in SQL
+  }
+
  /**
   * Deterministic function that returns a bucket id based on the resource's roid.
   * NB: At the moment, nothing depends on this being deterministic, so we have the ability to
@@ -43,6 +43,8 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
 import java.util.Optional;
@@ -61,28 +63,44 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
  /** The {@link ForeignKeyIndex} type for {@link ContactResource} entities. */
  @ReportedOn
  @Entity
-  public static class ForeignKeyContactIndex extends ForeignKeyIndex<ContactResource> {}
+  public static class ForeignKeyContactIndex extends ForeignKeyIndex<ContactResource>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }

  /** The {@link ForeignKeyIndex} type for {@link DomainBase} entities. */
  @ReportedOn
  @Entity
-  public static class ForeignKeyDomainIndex extends ForeignKeyIndex<DomainBase> {}
+  public static class ForeignKeyDomainIndex extends ForeignKeyIndex<DomainBase>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }

  /** The {@link ForeignKeyIndex} type for {@link HostResource} entities. */
  @ReportedOn
  @Entity
-  public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource> {}
+  public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource>
+      implements DatastoreEntity {
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not relevant in SQL
+    }
+  }

-  static final ImmutableMap<
-          Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
+  static final ImmutableMap<Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
      RESOURCE_CLASS_TO_FKI_CLASS =
          ImmutableMap.of(
              ContactResource.class, ForeignKeyContactIndex.class,
              DomainBase.class, ForeignKeyDomainIndex.class,
              HostResource.class, ForeignKeyHostIndex.class);

-  @Id
-  String foreignKey;
+  @Id String foreignKey;

  /**
   * The deletion time of this {@link ForeignKeyIndex}.
@@ -90,8 +108,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   * <p>This will generally be equal to the deletion time of {@link #topReference}. However, in the
   * case of a {@link HostResource} that was renamed, this field will hold the time of the rename.
   */
-  @Index
-  DateTime deletionTime;
+  @Index DateTime deletionTime;

  /**
   * The referenced resource.
@@ -45,6 +45,7 @@ import google.registry.model.transfer.TransferResponse.ContactTransferResponse;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.util.List;
 import java.util.Optional;
 import javax.persistence.AttributeOverride;
@@ -92,7 +93,7 @@ import org.joda.time.DateTime;
      @javax.persistence.Index(columnList = "eventTime")
    })
 public abstract class PollMessage extends ImmutableObject
-    implements Buildable, TransferServerApproveEntity {
+    implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity {

  /** Entity id. */
  @Id
@@ -340,12 +340,12 @@ public class Registry extends ImmutableObject implements Buildable {
  TldType tldType = TldType.REAL;

  /**
-   * Whether to disable invoicing for a {@link TldType#REAL} TLD.
+   * Whether to enable invoicing for this TLD.
   *
-   * <p>Note that invoicing is always disabled for {@link TldType#TEST} TLDs. Setting this field has
-   * no effect for {@link TldType#TEST} TLDs.
+   * <p>Note that this boolean is the sole determiner on whether invoices should be generated for a
+   * TLD. This applies to {@link TldType#TEST} TLDs as well.
   */
-  boolean disableInvoicing = false;
+  boolean invoicingEnabled = false;

  /**
   * A property that transitions to different TldStates at different times. Stored as a list of
@@ -646,8 +646,8 @@ public class Registry extends ImmutableObject implements Buildable {
      return this;
    }

-    public Builder setDisableInvoicing(boolean disableInvoicing) {
-      getInstance().disableInvoicing = disableInvoicing;
+    public Builder setInvoicingEnabled(boolean invoicingEnabled) {
+      getInstance().invoicingEnabled = invoicingEnabled;
      return this;
    }

@@ -114,7 +114,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
  /** Virtual parent entity for premium list entry entities associated with a single revision. */
  @ReportedOn
  @Entity
-  public static class PremiumListRevision extends ImmutableObject {
+  public static class PremiumListRevision extends ImmutableObject implements DatastoreEntity {

    @Parent Key<PremiumList> parent;

@@ -171,6 +171,11 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
      }
      return revision;
    }
+
+    @Override
+    public ImmutableList<SqlEntity> toSqlEntities() {
+      return ImmutableList.of(); // not persisted in SQL
+    }
  }

  /**
@@ -18,6 +18,7 @@ import static com.googlecode.objectify.Key.getKind;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;

 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -40,17 +41,18 @@ import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import java.util.Set;
 import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
 import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
 import javax.persistence.MappedSuperclass;
-import javax.persistence.SequenceGenerator;
 import javax.persistence.Transient;
 import org.joda.time.DateTime;

@@ -59,7 +61,8 @@ import org.joda.time.DateTime;
@Entity
@MappedSuperclass
@WithStringVKey // TODO(b/162229294): This should be resolved during the course of that bug
-public class HistoryEntry extends ImmutableObject implements Buildable {
+@Access(AccessType.FIELD)
+public class HistoryEntry extends ImmutableObject implements Buildable, DatastoreEntity, SqlEntity {

  /** Represents the type of history entry. */
  public enum Type {
@@ -110,17 +113,13 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
    SYNTHETIC
  }

-  /** The autogenerated id of this event. */
-  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "HistorySequenceGenerator")
-  @SequenceGenerator(
-      name = "HistorySequenceGenerator",
-      sequenceName = "history_id_sequence",
-      allocationSize = 1)
-  @Id
-  @javax.persistence.Id
-  @Column(name = "historyRevisionId")
-  @VisibleForTesting
-  public Long id;
+  /**
+   * The autogenerated id of this event. Note that, this field is marked as {@link Transient} in the
+   * SQL schema, this is because the child class of {@link HistoryEntry}, e.g. {@link
+   * DomainHistory}, uses a composite primary key which the id is part of, and Hibernate requires
+   * that all the {@link javax.persistence.Id} fields must be put in the exact same class.
+   */
+  @Id @Transient @VisibleForTesting public Long id;

  /** The resource this event mutated. */
  @Parent @Transient protected Key<? extends EppResource> parent;
@@ -196,8 +195,17 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
  @Transient // domain-specific
  Set<DomainTransactionRecord> domainTransactionRecords;

-  public Long getId() {
-    return id;
+  public long getId() {
+    // For some reason, Hibernate throws NPE during some initialization phase if we don't deal with
+    // the null case. Setting the id to 0L when it is null should be fine because 0L for primitive
+    // type is considered as null for wrapper class in the Hibernate context.
+    return id == null ? 0L : id;
+  }
+
+  // This method is required by Hibernate.
+  @SuppressWarnings("UnusedMethod")
+  private void setId(long id) {
+    this.id = id;
  }

  public Key<? extends EppResource> getParent() {
@@ -279,11 +287,7 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
    // can't use a switch statement since we're calling getKind()
    if (parentKind.equals(getKind(DomainBase.class))) {
      resultEntity =
-          new DomainHistory.Builder()
-              .copyFrom(this)
-              .setDomainRepoId(
-                  VKey.create(DomainBase.class, parent.getName(), (Key<DomainBase>) parent))
-              .build();
+          new DomainHistory.Builder().copyFrom(this).setDomainRepoId(parent.getName()).build();
    } else if (parentKind.equals(getKind(HostResource.class))) {
      resultEntity =
          new HostHistory.Builder()
@@ -306,6 +310,18 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
    return resultEntity;
  }

+  // In SQL, save the child type
+  @Override
+  public ImmutableList<SqlEntity> toSqlEntities() {
+    return ImmutableList.of(toChildHistoryEntity());
+  }
+
+  // In Datastore, save as a HistoryEntry object regardless of this object's type
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(asHistoryEntry());
+  }
+
  /** A builder for {@link HistoryEntry} since it is immutable */
  public static class Builder<T extends HistoryEntry, B extends Builder<?, ?>>
      extends GenericBuilder<T, B> {
@@ -349,6 +365,7 @@ public class HistoryEntry extends ImmutableObject implements Buildable {
      return thisCastToDerived();
    }

+    // Until we move completely to SQL, override this in subclasses (e.g. HostHistory) to set VKeys
    public B setParent(Key<? extends EppResource> parent) {
      getInstance().parent = parent;
      return thisCastToDerived();
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-package google.registry.schema.tmch;
+package google.registry.model.tmch;

 import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
 import static google.registry.model.CacheUtils.tryMemoizeWithExpiration;
@@ -24,28 +24,28 @@ import google.registry.util.NonFinalForTesting;
 import java.util.Optional;
 import javax.persistence.EntityManager;

-/** Data access object for {@link ClaimsList}. */
+/** Data access object for {@link ClaimsListShard}. */
 public class ClaimsListDao {

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

  /** In-memory cache for claims list. */
  @NonFinalForTesting
-  private static Supplier<Optional<ClaimsList>> cacheClaimsList =
+  private static Supplier<Optional<ClaimsListShard>> cacheClaimsList =
      tryMemoizeWithExpiration(getDomainLabelListCacheDuration(), ClaimsListDao::getLatestRevision);

-  private static void save(ClaimsList claimsList) {
+  private static void save(ClaimsListShard claimsList) {
    jpaTm().transact(() -> jpaTm().getEntityManager().persist(claimsList));
  }

  /**
-   * Try to save the given {@link ClaimsList} into Cloud SQL. If the save fails, the error will be
-   * logged but no exception will be thrown.
+   * Try to save the given {@link ClaimsListShard} into Cloud SQL. If the save fails, the error will
+   * be logged but no exception will be thrown.
   *
   * <p>This method is used during the dual-write phase of database migration as Datastore is still
   * the authoritative database.
   */
-  public static void trySave(ClaimsList claimsList) {
+  static void trySave(ClaimsListShard claimsList) {
    try {
      ClaimsListDao.save(claimsList);
      logger.atInfo().log(
@@ -57,12 +57,12 @@ public class ClaimsListDao {
  }

  /**
-   * Returns the most recent revision of the {@link ClaimsList} in Cloud SQL, if it exists.
+   * Returns the most recent revision of the {@link ClaimsListShard} in Cloud SQL, if it exists.
   * TODO(shicong): Change this method to package level access after dual-read phase.
   * ClaimsListShard uses this method to retrieve claims list in Cloud SQL for the comparison, and
   * ClaimsListShard is not in this package.
   */
-  public static Optional<ClaimsList> getLatestRevision() {
+  public static Optional<ClaimsListShard> getLatestRevision() {
    return jpaTm()
        .transact(
            () -> {
@@ -73,15 +73,15 @@ public class ClaimsListDao {
              return em.createQuery(
                      "FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
                          + " :revisionId",
-                      ClaimsList.class)
+                      ClaimsListShard.class)
                  .setParameter("revisionId", revisionId)
                  .getResultStream()
                  .findFirst();
            });
  }

-  /** Returns the most recent revision of the {@link ClaimsList}, from cache. */
-  public static Optional<ClaimsList> getLatestRevisionCached() {
+  /** Returns the most recent revision of the {@link ClaimsListShard}, from cache. */
+  public static Optional<ClaimsListShard> getLatestRevisionCached() {
    return cacheClaimsList.get();
  }

@@ -40,6 +40,7 @@ import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.OnSave;
 import com.googlecode.objectify.annotation.Parent;
+import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
@@ -47,8 +48,6 @@ import google.registry.model.annotations.VirtualEntity;
 import google.registry.model.common.CrossTldSingleton;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
-import google.registry.schema.tmch.ClaimsList;
-import google.registry.schema.tmch.ClaimsListDao;
 import google.registry.util.CollectionUtils;
 import google.registry.util.Concurrent;
 import google.registry.util.Retrier;
@@ -59,6 +58,15 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.Callable;
 import javax.annotation.Nullable;
+import javax.persistence.CollectionTable;
+import javax.persistence.Column;
+import javax.persistence.ElementCollection;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.JoinColumn;
+import javax.persistence.MapKeyColumn;
+import javax.persistence.Table;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;

 /**
@@ -74,10 +82,21 @@ import org.joda.time.DateTime;
 * 10MB per transaction limit.
 *
 * <p>Therefore, it is never OK to save an instance of this class directly to Datastore. Instead you
- * must use the {@link #save} method to do it for you.
+ * must use the {@link #saveToDatastore} method to do it for you.
+ *
+ * <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
+ * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
+ * succeeds, we will end up with having two exact same claims list with only different {@link
+ * #revisionId}. However, this is not an actual problem because we only use the claims list with
+ * highest {@link #revisionId}.
+ *
+ * <p>TODO(b/162007765): Rename the class to ClaimsList and remove Datastore related fields and
+ * methods.
 */
@Entity
@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
+@javax.persistence.Entity(name = "ClaimsList")
+@Table
 public class ClaimsListShard extends ImmutableObject implements DatastoreEntity {

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -85,22 +104,44 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
  /** The number of claims list entries to store per shard. */
  private static final int SHARD_SIZE = 10000;

-  @Id
-  long id;
+  @Transient @Id long id;

-  @Parent
-  Key<ClaimsListRevision> parent;
+  @Transient @Parent Key<ClaimsListRevision> parent;

-  /** When the claims list was last updated. */
+  @Ignore
+  @javax.persistence.Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  Long revisionId;
+
+  @Ignore
+  @Column(nullable = false)
+  CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
+
+  /**
+   * When the claims list was last updated.
+   *
+   * <p>Note that the value of this field is parsed from the claims list file(See this <a
+   * href="https://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.1">RFC</>), it is
+   * the DNL List creation datetime from the rfc. Since this field has been used by Datastore, we
+   * cannot change its name until we finish the migration.
+   *
+   * <p>TODO(b/166784536): Rename this field to tmdbGenerationTime.
+   */
+  @Column(name = "tmdb_generation_time", nullable = false)
  DateTime creationTime;

  /** A map from labels to claims keys. */
  @EmbedMap
+  @ElementCollection
+  @CollectionTable(
+      name = "ClaimsEntry",
+      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
+  @MapKeyColumn(name = "domainLabel", nullable = false)
+  @Column(name = "claimKey", nullable = false)
  Map<String, String> labelsToKeys;

  /** Indicates that this is a shard rather than a "full" list. */
-  @Ignore
-  boolean isShard = false;
+  @Ignore @Transient boolean isShard = false;

  private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2);

@@ -164,10 +205,10 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
        return datastoreList;
      };

-  private static final void loadAndCompareCloudSqlList(ClaimsListShard datastoreList) {
-    Optional<ClaimsList> maybeCloudSqlList = ClaimsListDao.getLatestRevision();
+  private static void loadAndCompareCloudSqlList(ClaimsListShard datastoreList) {
+    Optional<ClaimsListShard> maybeCloudSqlList = ClaimsListDao.getLatestRevision();
    if (maybeCloudSqlList.isPresent()) {
-      ClaimsList cloudSqlList = maybeCloudSqlList.get();
+      ClaimsListShard cloudSqlList = maybeCloudSqlList.get();
      MapDifference<String, String> diff =
          Maps.difference(datastoreList.labelsToKeys, cloudSqlList.getLabelsToKeys());
      if (!diff.areEqual()) {
@@ -206,15 +247,34 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
      memoizeWithShortExpiration(
          () -> LOADER_RETRIER.callWithRetry(LOADER_CALLABLE, IllegalStateException.class));

-  public DateTime getCreationTime() {
+  /** Returns the revision id of this claims list, or throws exception if it is null. */
+  public Long getRevisionId() {
+    checkState(
+        revisionId != null, "revisionId is null because it is not persisted in the database");
+    return revisionId;
+  }
+
+  /**
+   * Returns the time when the external TMDB service generated this revision of the claims list.
+   *
+   * @see <a href="https://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.1">DNL List
+   *     creation datetime</a>
+   */
+  public DateTime getTmdbGenerationTime() {
    return creationTime;
  }

+  /** Returns the creation time of this claims list. */
+  public DateTime getCreationTimestamp() {
+    return creationTimestamp.getTimestamp();
+  }
+
  /** Returns the claim key for a given domain if there is one, empty otherwise. */
  public Optional<String> getClaimKey(String label) {
    return Optional.ofNullable(labelsToKeys.get(label));
  }

+  /** Returns an {@link Map} mapping domain label to its lookup key. */
  public ImmutableMap<String, String> getLabelsToKeys() {
    return ImmutableMap.copyOf(labelsToKeys);
  }
@@ -229,11 +289,12 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
   * switching over to using them atomically, then deleting the old ones.
   */
  public void save() {
-    save(SHARD_SIZE);
+    saveToDatastore(SHARD_SIZE);
+    ClaimsListDao.trySave(this);
  }

  @VisibleForTesting
-  void save(int shardSize) {
+  void saveToDatastore(int shardSize) {
    // Figure out what the next versionId should be based on which ones already exist.
    final Key<ClaimsListRevision> oldRevision = getCurrentRevision();
    final Key<ClaimsListRevision> parentKey = ClaimsListRevision.createKey();
@@ -270,10 +331,11 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreEntity
            });
  }

-  public static ClaimsListShard create(DateTime creationTime, Map<String, String> labelsToKeys) {
+  public static ClaimsListShard create(
+      DateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
    ClaimsListShard instance = new ClaimsListShard();
    instance.id = allocateId();
-    instance.creationTime = checkNotNull(creationTime);
+    instance.creationTime = checkNotNull(tmdbGenerationTime);
    instance.labelsToKeys = checkNotNull(labelsToKeys);
    return instance;
  }
@@ -0,0 +1,37 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.registry.label.PremiumList;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA converter for a {@link Key} containing a {@link PremiumList} */
+@Converter(autoApply = true)
+public class PremiumListKeyConverter implements AttributeConverter<Key<PremiumList>, String> {
+
+  @Override
+  public String convertToDatabaseColumn(Key<PremiumList> attribute) {
+    return (attribute == null) ? null : attribute.getName();
+  }
+
+  @Override
+  public Key<PremiumList> convertToEntityAttribute(String dbData) {
+    return (dbData == null) ? null : Key.create(getCrossTldKey(), PremiumList.class, dbData);
+  }
+}
@@ -0,0 +1,37 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.registry.label.ReservedList;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA converter for a {@link Key} containing a {@link ReservedList} */
+@Converter(autoApply = true)
+public class ReservedListKeyConverter implements AttributeConverter<Key<ReservedList>, String> {
+
+  @Override
+  public String convertToDatabaseColumn(Key<ReservedList> attribute) {
+    return (attribute == null) ? null : attribute.getName();
+  }
+
+  @Override
+  public Key<ReservedList> convertToEntityAttribute(String dbData) {
+    return (dbData == null) ? null : Key.create(getCrossTldKey(), ReservedList.class, dbData);
+  }
+}
@@ -391,7 +391,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
  private static ImmutableSet<EntityId> getEntityIdsFromEntity(
      EntityType<?> entityType, Object entity) {
    if (entityType.hasSingleIdAttribute()) {
-      String idName = entityType.getDeclaredId(entityType.getIdType().getJavaType()).getName();
+      String idName = entityType.getId(entityType.getIdType().getJavaType()).getName();
      Object idValue = getFieldValue(entity, idName);
      return ImmutableSet.of(new EntityId(idName, idValue));
    } else {
@@ -402,7 +402,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
  private static ImmutableSet<EntityId> getEntityIdsFromSqlKey(
      EntityType<?> entityType, Object sqlKey) {
    if (entityType.hasSingleIdAttribute()) {
-      String idName = entityType.getDeclaredId(entityType.getIdType().getJavaType()).getName();
+      String idName = entityType.getId(entityType.getIdType().getJavaType()).getName();
      return ImmutableSet.of(new EntityId(idName, sqlKey));
    } else {
      return getEntityIdsFromIdContainer(entityType, sqlKey);
@@ -429,7 +429,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {

  private static Object getFieldValue(Object object, String fieldName) {
    try {
-      Field field = object.getClass().getDeclaredField(fieldName);
+      Field field = getField(object.getClass(), fieldName);
      field.setAccessible(true);
      return field.get(object);
    } catch (NoSuchFieldException | IllegalAccessException e) {
@@ -437,6 +437,21 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
    }
  }

+  /** Gets the field definition from clazz or any superclass. */
+  private static Field getField(Class clazz, String fieldName) throws NoSuchFieldException {
+    try {
+      // Note that we have to use getDeclaredField() for this, getField() just finds public fields.
+      return clazz.getDeclaredField(fieldName);
+    } catch (NoSuchFieldException e) {
+      Class base = clazz.getSuperclass();
+      if (base != null) {
+        return getField(base, fieldName);
+      } else {
+        throw e;
+      }
+    }
+  }
+
  private static class TransactionInfo {
    EntityManager entityManager;
    boolean inTransaction = false;
@@ -14,6 +14,9 @@

 package google.registry.persistence.transaction;

+import com.google.common.collect.ImmutableList;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.SqlEntity;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
@@ -27,7 +30,7 @@ import javax.persistence.Table;
 */
@Entity
@Table(name = "Transaction")
-public class TransactionEntity {
+public class TransactionEntity implements SqlEntity {

  @Id
  @GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -40,4 +43,9 @@ public class TransactionEntity {
  TransactionEntity(byte[] contents) {
    this.contents = contents;
  }
+
+  @Override
+  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
+    return ImmutableList.of(); // not stored in Datastore per se
+  }
 }
@@ -24,6 +24,8 @@ import com.google.api.client.http.GenericUrl;
 import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.http.HttpRequest;
 import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
 import com.google.api.client.http.HttpTransport;
 import com.google.common.base.Ascii;
 import com.google.common.base.Splitter;
@@ -80,6 +82,7 @@ public class IcannHttpReporter {
    headers.setContentType(CSV_UTF_8.toString());
    request.setHeaders(headers);
    request.setFollowRedirects(false);
+    request.setThrowExceptionOnExecuteError(false);

    HttpResponse response = null;
    logger.atInfo().log(
@@ -87,6 +90,12 @@ public class IcannHttpReporter {
    boolean success = true;
    try {
      response = request.execute();
+      // Only responses with a 200 or 400 status have a body. For everything else, throw so that
+      // the caller catches it and prints the stack trace.
+      if (response.getStatusCode() != HttpStatusCodes.STATUS_CODE_OK
+          && response.getStatusCode() != HttpStatusCodes.STATUS_CODE_BAD_REQUEST) {
+        throw new HttpResponseException(response);
+      }
      byte[] content;
      try {
        content = ByteStreams.toByteArray(response.getContent());
@@ -94,13 +103,24 @@ public class IcannHttpReporter {
        response.getContent().close();
      }
      logger.atInfo().log(
-          "Received response code %d with content: %s\n\nResponse content in hex: %s",
+          "Received response code %d\n\n"
+              + "Response headers: %s\n\n"
+              + "Response content in UTF-8: %s\n\n"
+              + "Response content in HEX: %s",
          response.getStatusCode(),
+          response.getHeaders(),
          new String(content, UTF_8),
          BaseEncoding.base16().encode(content));
-      XjcIirdeaResult result = parseResult(content);
-      if (result.getCode().getValue() != 1000) {
+      // For reasons unclear at the moment, when we parse the response content using UTF-8 we get
+      // garbled texts. Since we know that an HTTP 200 response can only contain a result code of
+      // 1000 (i. e. success), there is no need to parse it.
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_BAD_REQUEST) {
        success = false;
+        // To debug if there is a problem with our parsing, we wrap the response and print the stack
+        // trace of it. As far as we can tell, the stack trace for such an exception contains the
+        // response content that is decoded correctly using the expected charset.
+        new HttpResponseException(response).printStackTrace();
+        XjcIirdeaResult result = parseResult(content);
        logger.atWarning().log(
            "PUT rejected, status code %s:\n%s\n%s",
            result.getCode(), result.getMsg(), result.getDescription());
@@ -1,116 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.schema.tmch;
-
-import static com.google.common.base.Preconditions.checkState;
-import static google.registry.util.DateTimeUtils.toJodaDateTime;
-import static google.registry.util.DateTimeUtils.toZonedDateTime;
-
-import com.google.common.collect.ImmutableList;
-import google.registry.model.CreateAutoTimestamp;
-import google.registry.model.ImmutableObject;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
-import java.time.ZonedDateTime;
-import java.util.Map;
-import java.util.Optional;
-import javax.persistence.CollectionTable;
-import javax.persistence.Column;
-import javax.persistence.ElementCollection;
-import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.MapKeyColumn;
-import javax.persistence.Table;
-import org.joda.time.DateTime;
-
-/**
- * A list of TMCH claims labels and their associated claims keys.
- *
- * <p>Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by
- * the database. So, if a retry of insertion happens after the previous attempt unexpectedly
- * succeeds, we will end up with having two exact same claims list with only different {@link
- * #revisionId}. However, this is not an actual problem because we only use the claims list with
- * highest {@link #revisionId}.
- */
-@Entity
-@Table
-public class ClaimsList extends ImmutableObject implements SqlEntity {
-  @Id
-  @GeneratedValue(strategy = GenerationType.IDENTITY)
-  @Column
-  private Long revisionId;
-
-  @Column(nullable = false)
-  private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
-
-  @Column(nullable = false)
-  private ZonedDateTime tmdbGenerationTime;
-
-  @ElementCollection
-  @CollectionTable(
-      name = "ClaimsEntry",
-      joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId"))
-  @MapKeyColumn(name = "domainLabel", nullable = false)
-  @Column(name = "claimKey", nullable = false)
-  private Map<String, String> labelsToKeys;
-
-  private ClaimsList(ZonedDateTime tmdbGenerationTime, Map<String, String> labelsToKeys) {
-    this.tmdbGenerationTime = tmdbGenerationTime;
-    this.labelsToKeys = labelsToKeys;
-  }
-
-  // Hibernate requires this default constructor.
-  private ClaimsList() {}
-
-  /** Constructs a {@link ClaimsList} object. */
-  public static ClaimsList create(DateTime creationTimestamp, Map<String, String> labelsToKeys) {
-    return new ClaimsList(toZonedDateTime(creationTimestamp), labelsToKeys);
-  }
-
-  /** Returns the revision id of this claims list, or throws exception if it is null. */
-  public Long getRevisionId() {
-    checkState(
-        revisionId != null, "revisionId is null because it is not persisted in the database");
-    return revisionId;
-  }
-
-  /** Returns the TMDB generation time of this claims list. */
-  public DateTime getTmdbGenerationTime() {
-    return toJodaDateTime(tmdbGenerationTime);
-  }
-
-  /** Returns the creation time of this claims list. */
-  public DateTime getCreationTimestamp() {
-    return creationTimestamp.getTimestamp();
-  }
-
-  /** Returns an {@link Map} mapping domain label to its lookup key. */
-  public Map<String, String> getLabelsToKeys() {
-    return labelsToKeys;
-  }
-
-  /** Returns the claim key for a given domain if there is one, empty otherwise. */
-  public Optional<String> getClaimKey(String label) {
-    return Optional.ofNullable(labelsToKeys.get(label));
-  }
-
-  @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // ClaimsList is dual-written
-  }
-}
@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;

 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableMap;
-import google.registry.schema.tmch.ClaimsList;
+import google.registry.model.tmch.ClaimsListShard;
 import java.util.List;
 import org.joda.time.DateTime;

@@ -34,11 +34,11 @@ import org.joda.time.DateTime;
 public class ClaimsListParser {

  /**
-   * Converts the lines from the DNL CSV file into a {@link ClaimsList} object.
+   * Converts the lines from the DNL CSV file into a {@link ClaimsListShard} object.
   *
   * <p>Please note that this does <b>not</b> insert the object into Datastore.
   */
-  public static ClaimsList parse(List<String> lines) {
+  public static ClaimsListShard parse(List<String> lines) {
    ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();

    // First line: <version>,<DNL List creation datetime>
@@ -74,6 +74,6 @@ public class ClaimsListParser {
      builder.put(label, lookupKey);
    }

-    return ClaimsList.create(creationTime, builder.build());
+    return ClaimsListShard.create(creationTime, builder.build());
  }
 }
@@ -21,8 +21,6 @@ import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.tmch.ClaimsListShard;
 import google.registry.request.Action;
 import google.registry.request.auth.Auth;
-import google.registry.schema.tmch.ClaimsList;
-import google.registry.schema.tmch.ClaimsListDao;
 import java.io.IOException;
 import java.security.SignatureException;
 import java.util.List;
@@ -56,14 +54,10 @@ public final class TmchDnlAction implements Runnable {
    } catch (SignatureException | IOException | PGPException e) {
      throw new RuntimeException(e);
    }
-    ClaimsList claims = ClaimsListParser.parse(lines);
-    ClaimsListShard claimsListShard =
-        ClaimsListShard.create(claims.getTmdbGenerationTime(), claims.getLabelsToKeys());
-    claimsListShard.save();
+    ClaimsListShard claims = ClaimsListParser.parse(lines);
+    claims.save();
    logger.atInfo().log(
        "Inserted %,d claims into Datastore, created at %s",
-        claimsListShard.size(), claimsListShard.getCreationTime());
-
-    ClaimsListDao.trySave(claims);
+        claims.size(), claims.getTmdbGenerationTime());
  }
 }
@@ -118,10 +118,10 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {

  @Nullable
  @Parameter(
-      names = "--disable_invoicing",
-      description = "Whether invoicing is disabled for a REAL tld.",
+      names = "--invoicing_enabled",
+      description = "Whether invoicing is enabled for this tld.",
      arity = 1)
-  private Boolean disableInvoicing;
+  private Boolean invoicingEnabled;

  @Nullable
  @Parameter(
@@ -327,7 +327,7 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
      Optional.ofNullable(serverStatusChangeCost)
          .ifPresent(builder::setServerStatusChangeBillingCost);
      Optional.ofNullable(tldType).ifPresent(builder::setTldType);
-      Optional.ofNullable(disableInvoicing).ifPresent(builder::setDisableInvoicing);
+      Optional.ofNullable(invoicingEnabled).ifPresent(builder::setInvoicingEnabled);
      Optional.ofNullable(lordnUsername).ifPresent(u -> builder.setLordnUsername(u.orElse(null)));
      Optional.ofNullable(claimsPeriodEnd).ifPresent(builder::setClaimsPeriodEnd);
      Optional.ofNullable(numDnsPublishShards).ifPresent(builder::setNumDnsPublishLocks);
@@ -22,8 +22,6 @@ import com.beust.jcommander.Parameters;
 import com.google.common.base.Joiner;
 import com.google.common.io.Files;
 import google.registry.model.tmch.ClaimsListShard;
-import google.registry.schema.tmch.ClaimsList;
-import google.registry.schema.tmch.ClaimsListDao;
 import google.registry.tmch.ClaimsListParser;
 import java.io.File;
 import java.io.IOException;
@@ -39,7 +37,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command

  private String claimsListFilename;

-  private ClaimsList claimsList;
+  private ClaimsListShard claimsList;

  @Override
  protected void init() throws IOException {
@@ -58,8 +56,7 @@ final class UploadClaimsListCommand extends ConfirmingCommand implements Command

  @Override
  public String execute() {
-    ClaimsListShard.create(claimsList.getTmdbGenerationTime(), claimsList.getLabelsToKeys()).save();
-    ClaimsListDao.trySave(claimsList);
+    claimsList.save();
    return String.format("Successfully uploaded claims list %s", claimsListFilename);
  }
 }
@@ -10,6 +10,12 @@
      <basic name="amount" access="FIELD"/>
    </attributes>
  </embeddable>
+
+  <sequence-generator name="HistorySequenceGenerator" sequence-name="history_id_sequence"/>
+
+  <!-- TODO(shicong): Drop this sequence and change all history tables to use the above one. -->
+  <sequence-generator name="TempHistorySequenceGenerator" sequence-name="temp_history_id_sequence"/>
+
  <persistence-unit-metadata>
    <persistence-unit-defaults>
      <entity-listeners>
@@ -53,8 +53,8 @@
    <class>google.registry.model.registry.label.PremiumList</class>
    <class>google.registry.model.reporting.Spec11ThreatMatch</class>
    <class>google.registry.persistence.transaction.TransactionEntity</class>
+    <class>google.registry.model.tmch.ClaimsListShard</class>
    <class>google.registry.schema.domain.RegistryLock</class>
-    <class>google.registry.schema.tmch.ClaimsList</class>
    <class>google.registry.schema.cursor.Cursor</class>
    <class>google.registry.schema.server.Lock</class>
    <class>google.registry.schema.tld.PremiumEntry</class>
@@ -79,7 +79,9 @@
    <class>google.registry.persistence.converter.InetAddressSetConverter</class>
    <class>google.registry.persistence.converter.LocalDateConverter</class>
    <class>google.registry.persistence.converter.PostalInfoChoiceListConverter</class>
+    <class>google.registry.persistence.converter.PremiumListKeyConverter</class>
    <class>google.registry.persistence.converter.RegistrarPocSetConverter</class>
+    <class>google.registry.persistence.converter.ReservedListKeyConverter</class>
    <class>google.registry.persistence.converter.Spec11ThreatMatchThreatTypeSetConverter</class>
    <class>google.registry.persistence.converter.StatusValueSetConverter</class>
    <class>google.registry.persistence.converter.StringListConverter</class>
@@ -21,6 +21,7 @@ import static org.joda.time.DateTimeZone.UTC;

 import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -38,6 +39,7 @@ public class CreateAutoTimestampTest {

  /** Timestamped class. */
  @Entity(name = "CatTestEntity")
+  @EntityForTesting
  public static class TestObject extends CrossTldSingleton {
    CreateAutoTimestamp createTime = CreateAutoTimestamp.create(null);
  }
@@ -29,6 +29,7 @@ import com.google.common.collect.Iterables;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
+import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
 import google.registry.util.CidrAddressBlock;
 import java.util.ArrayDeque;
@@ -279,6 +280,7 @@ public class ImmutableObjectTest {

  /** Simple subclass of ImmutableObject. */
  @Entity
+  @EntityForTesting
  public static class ValueObject extends ImmutableObject {
    @Id
    long id;
@@ -21,6 +21,7 @@ import static org.joda.time.DateTimeZone.UTC;

 import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -38,6 +39,7 @@ public class UpdateAutoTimestampTest {

  /** Timestamped class. */
  @Entity(name = "UatTestEntity")
+  @EntityForTesting
  public static class UpdateAutoTimestampTestObject extends CrossTldSingleton {
    UpdateAutoTimestamp updateTime = UpdateAutoTimestamp.create(null);
  }
@@ -14,6 +14,7 @@

 package google.registry.model.history;

+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
@@ -71,11 +72,15 @@ public class DomainHistoryTest extends EntityTestCase {
    jpaTm()
        .transact(
            () -> {
-              DomainHistory fromDatabase =
-                  jpaTm().load(VKey.createSql(DomainHistory.class, domainHistory.getId()));
+              DomainHistory fromDatabase = jpaTm().load(domainHistory.createVKey());
              assertDomainHistoriesEqual(fromDatabase, domainHistory);
              assertThat(fromDatabase.getDomainRepoId().getSqlKey())
                  .isEqualTo(domainHistory.getDomainRepoId().getSqlKey());
+              assertThat(fromDatabase.getNsHosts())
+                  .containsExactlyElementsIn(
+                      domainHistory.getNsHosts().stream()
+                          .map(key -> VKey.createSql(HostResource.class, key.getSqlKey()))
+                          .collect(toImmutableSet()));
            });
  }

@@ -120,7 +125,7 @@ public class DomainHistoryTest extends EntityTestCase {
  static void assertDomainHistoriesEqual(DomainHistory one, DomainHistory two) {
    assertAboutImmutableObjects()
        .that(one)
-        .isEqualExceptFields(two, "domainContent", "domainRepoId", "parent");
+        .isEqualExceptFields(two, "domainContent", "domainRepoId", "parent", "nsHosts");
  }

  private DomainHistory createDomainHistory(DomainContent domain) {
@@ -134,7 +139,7 @@ public class DomainHistoryTest extends EntityTestCase {
        .setReason("reason")
        .setRequestedByRegistrar(true)
        .setDomainContent(domain)
-        .setDomainRepoId(domain.createVKey())
+        .setDomainRepoId(domain.getRepoId())
        .build();
  }
 }
@@ -45,6 +45,7 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DatastoreHelper;
 import google.registry.testing.FakeClock;
@@ -69,13 +70,14 @@ public class OfyTest {
  @BeforeEach
  void beforeEach() {
    createTld("tld");
-    someObject = new HistoryEntry.Builder()
-        .setClientId("client id")
-        .setModificationTime(START_OF_TIME)
-        .setParent(persistActiveContact("parentContact"))
-        .setTrid(Trid.create("client", "server"))
-        .setXmlBytes("<xml></xml>".getBytes(UTF_8))
-        .build();
+    someObject =
+        new HistoryEntry.Builder()
+            .setClientId("client id")
+            .setModificationTime(START_OF_TIME)
+            .setParent(persistActiveContact("parentContact"))
+            .setTrid(Trid.create("client", "server"))
+            .setXmlBytes("<xml></xml>".getBytes(UTF_8))
+            .build();
    // This can't be initialized earlier because namespaces need the AppEngineRule to work.
  }

@@ -111,8 +113,7 @@ public class OfyTest {
        assertThrows(
            IllegalArgumentException.class,
            () ->
-                tm()
-                    .transact(
+                tm().transact(
                        () -> {
                          ofy().save().entity(someObject);
                          ofy().save().entity(someObject);
@@ -126,8 +127,7 @@ public class OfyTest {
        assertThrows(
            IllegalArgumentException.class,
            () ->
-                tm()
-                    .transact(
+                tm().transact(
                        () -> {
                          ofy().delete().entity(someObject);
                          ofy().delete().entity(someObject);
@@ -141,8 +141,7 @@ public class OfyTest {
        assertThrows(
            IllegalArgumentException.class,
            () ->
-                tm()
-                    .transact(
+                tm().transact(
                        () -> {
                          ofy().save().entity(someObject);
                          ofy().delete().entity(someObject);
@@ -156,8 +155,7 @@ public class OfyTest {
        assertThrows(
            IllegalArgumentException.class,
            () ->
-                tm()
-                    .transact(
+                tm().transact(
                        () -> {
                          ofy().delete().entity(someObject);
                          ofy().save().entity(someObject);
@@ -174,13 +172,12 @@ public class OfyTest {

  /** Simple entity class with lifecycle callbacks. */
  @com.googlecode.objectify.annotation.Entity
+  @EntityForTesting
  public static class LifecycleObject extends ImmutableObject {

-    @Parent
-    Key<?> parent = getCrossTldKey();
+    @Parent Key<?> parent = getCrossTldKey();

-    @Id
-    long id = 1;
+    @Id long id = 1;

    boolean onLoadCalled;
    boolean onSaveCalled;
@@ -218,20 +215,22 @@ public class OfyTest {
  /** Avoid regressions of b/21309102 where transaction time did not change on each retry. */
  @Test
  void testTransact_getsNewTimestampOnEachTry() {
-    tm().transact(new Runnable() {
+    tm().transact(
+            new Runnable() {

-      DateTime firstAttemptTime;
+              DateTime firstAttemptTime;

-      @Override
-      public void run() {
-        if (firstAttemptTime == null) {
-          // Sleep a bit to ensure that the next attempt is at a new millisecond.
-          firstAttemptTime = tm().getTransactionTime();
-          sleepUninterruptibly(10, MILLISECONDS);
-          throw new ConcurrentModificationException();
-        }
-        assertThat(tm().getTransactionTime()).isGreaterThan(firstAttemptTime);
-      }});
+              @Override
+              public void run() {
+                if (firstAttemptTime == null) {
+                  // Sleep a bit to ensure that the next attempt is at a new millisecond.
+                  firstAttemptTime = tm().getTransactionTime();
+                  sleepUninterruptibly(10, MILLISECONDS);
+                  throw new ConcurrentModificationException();
+                }
+                assertThat(tm().getTransactionTime()).isGreaterThan(firstAttemptTime);
+              }
+            });
  }

  @Test
@@ -319,17 +318,19 @@ public class OfyTest {
          }
        };
    // A commit logged work that throws on the first attempt to get its result.
-    CommitLoggedWork<Void> commitLoggedWork = new CommitLoggedWork<Void>(work, new SystemClock()) {
-      boolean firstCallToGetResult = true;
+    CommitLoggedWork<Void> commitLoggedWork =
+        new CommitLoggedWork<Void>(work, new SystemClock()) {
+          boolean firstCallToGetResult = true;

-      @Override
-      public Void getResult() {
-        if (firstCallToGetResult) {
-          firstCallToGetResult = false;
-          throw new DatastoreTimeoutException("");
-        }
-        return null;
-      }};
+          @Override
+          public Void getResult() {
+            if (firstCallToGetResult) {
+              firstCallToGetResult = false;
+              throw new DatastoreTimeoutException("");
+            }
+            return null;
+          }
+        };
    // Despite the DatastoreTimeoutException in the first call to getResult(), this should succeed
    // without retrying. If a retry is triggered, the test should fail due to the call to fail().
    ofy().transactCommitLoggedWork(commitLoggedWork);
@@ -381,8 +382,7 @@ public class OfyTest {
  void test_getBaseEntityClassFromEntityOrKey_subclassEntity() {
    DomainBase domain = DatastoreHelper.newDomainBase("test.tld");
    assertThat(getBaseEntityClassFromEntityOrKey(domain)).isEqualTo(DomainBase.class);
-    assertThat(getBaseEntityClassFromEntityOrKey(Key.create(domain)))
-        .isEqualTo(DomainBase.class);
+    assertThat(getBaseEntityClassFromEntityOrKey(Key.create(domain))).isEqualTo(DomainBase.class);
  }

  @Test
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-package google.registry.schema.tmch;
+package google.registry.model.tmch;

 import static com.google.common.truth.Truth.assertThat;

@@ -40,20 +40,22 @@ public class ClaimsListDaoTest {

  @Test
  void trySave_insertsClaimsListSuccessfully() {
-    ClaimsList claimsList =
-        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsListShard claimsList =
+        ClaimsListShard.create(
+            fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
    ClaimsListDao.trySave(claimsList);
-    ClaimsList insertedClaimsList = ClaimsListDao.getLatestRevision().get();
+    ClaimsListShard insertedClaimsList = ClaimsListDao.getLatestRevision().get();
    assertClaimsListEquals(claimsList, insertedClaimsList);
    assertThat(insertedClaimsList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc());
  }

  @Test
  void trySave_noExceptionThrownWhenSaveFail() {
-    ClaimsList claimsList =
-        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsListShard claimsList =
+        ClaimsListShard.create(
+            fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
    ClaimsListDao.trySave(claimsList);
-    ClaimsList insertedClaimsList = ClaimsListDao.getLatestRevision().get();
+    ClaimsListShard insertedClaimsList = ClaimsListDao.getLatestRevision().get();
    assertClaimsListEquals(claimsList, insertedClaimsList);
    // Save ClaimsList with existing revisionId should fail because revisionId is the primary key.
    ClaimsListDao.trySave(insertedClaimsList);
@@ -61,9 +63,9 @@ public class ClaimsListDaoTest {

  @Test
  void trySave_claimsListWithNoEntries() {
-    ClaimsList claimsList = ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of());
+    ClaimsListShard claimsList = ClaimsListShard.create(fakeClock.nowUtc(), ImmutableMap.of());
    ClaimsListDao.trySave(claimsList);
-    ClaimsList insertedClaimsList = ClaimsListDao.getLatestRevision().get();
+    ClaimsListShard insertedClaimsList = ClaimsListDao.getLatestRevision().get();
    assertClaimsListEquals(claimsList, insertedClaimsList);
    assertThat(insertedClaimsList.getLabelsToKeys()).isEmpty();
  }
@@ -75,16 +77,18 @@ public class ClaimsListDaoTest {

  @Test
  void getCurrent_returnsLatestClaims() {
-    ClaimsList oldClaimsList =
-        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
-    ClaimsList newClaimsList =
-        ClaimsList.create(fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
+    ClaimsListShard oldClaimsList =
+        ClaimsListShard.create(
+            fakeClock.nowUtc(), ImmutableMap.of("label1", "key1", "label2", "key2"));
+    ClaimsListShard newClaimsList =
+        ClaimsListShard.create(
+            fakeClock.nowUtc(), ImmutableMap.of("label3", "key3", "label4", "key4"));
    ClaimsListDao.trySave(oldClaimsList);
    ClaimsListDao.trySave(newClaimsList);
    assertClaimsListEquals(newClaimsList, ClaimsListDao.getLatestRevision().get());
  }

-  private void assertClaimsListEquals(ClaimsList left, ClaimsList right) {
+  private void assertClaimsListEquals(ClaimsListShard left, ClaimsListShard right) {
    assertThat(left.getRevisionId()).isEqualTo(right.getRevisionId());
    assertThat(left.getTmdbGenerationTime()).isEqualTo(right.getTmdbGenerationTime());
    assertThat(left.getLabelsToKeys()).isEqualTo(right.getLabelsToKeys());
@@ -76,7 +76,7 @@ public class ClaimsListShardTest {
    DateTime now = DateTime.now(UTC);
    // Save it with sharding, and make sure that reloading it works.
    ClaimsListShard unsharded = ClaimsListShard.create(now, ImmutableMap.copyOf(labelsToKeys));
-    unsharded.save(shardSize);
+    unsharded.saveToDatastore(shardSize);
    assertThat(ClaimsListShard.get().labelsToKeys).isEqualTo(unsharded.labelsToKeys);
    List<ClaimsListShard> shards1 = ofy().load().type(ClaimsListShard.class).list();
    assertThat(shards1).hasSize(4);
@@ -90,7 +90,7 @@ public class ClaimsListShardTest {
      labelsToKeys.put(Integer.toString(i), Integer.toString(i));
    }
    unsharded = ClaimsListShard.create(now.plusDays(1), ImmutableMap.copyOf(labelsToKeys));
-    unsharded.save(shardSize);
+    unsharded.saveToDatastore(shardSize);
    ofy().clearSessionCache();
    assertThat(ClaimsListShard.get().labelsToKeys).hasSize(unsharded.labelsToKeys.size());
    assertThat(ClaimsListShard.get().labelsToKeys).isEqualTo(unsharded.labelsToKeys);
@@ -27,6 +27,7 @@ import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.common.CrossTldSingleton;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.Ofy;
+import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectExtension;
@@ -42,6 +43,7 @@ public class CommitLogRevisionsTranslatorFactoryTest {
  private static final DateTime START_TIME = DateTime.parse("2000-01-01TZ");

  @Entity(name = "ClrtfTestEntity")
+  @EntityForTesting
  public static class TestObject extends CrossTldSingleton {
    ImmutableSortedMap<DateTime, Key<CommitLogManifest>> revisions = ImmutableSortedMap.of();
  }
@@ -73,11 +75,11 @@ public class CommitLogRevisionsTranslatorFactoryTest {

  @Test
  void testSave_doesNotMutateOriginalResource() {
-     TestObject object = new TestObject();
-     save(object);
-     assertThat(object.revisions).isEmpty();
-     assertThat(reload().revisions).isNotEmpty();
-   }
+    TestObject object = new TestObject();
+    save(object);
+    assertThat(object.revisions).isEmpty();
+    assertThat(reload().revisions).isNotEmpty();
+  }

  @Test
  void testSave_translatorAddsKeyToCommitLogToField() {
@@ -149,8 +151,10 @@ public class CommitLogRevisionsTranslatorFactoryTest {
    com.google.appengine.api.datastore.Entity entity =
        tm().transactNewReadOnly(() -> ofy().save().toEntity(reload()));
    assertThat(entity.getProperties().keySet()).containsExactly("revisions.key", "revisions.value");
-    assertThat(entity.getProperties()).containsEntry(
-        "revisions.key", ImmutableList.of(START_TIME.toDate(), START_TIME.plusDays(1).toDate()));
+    assertThat(entity.getProperties())
+        .containsEntry(
+            "revisions.key",
+            ImmutableList.of(START_TIME.toDate(), START_TIME.plusDays(1).toDate()));
    assertThat(entity.getProperty("revisions.value")).isInstanceOf(List.class);
    assertThat(((List<Object>) entity.getProperty("revisions.value")).get(0))
        .isInstanceOf(com.google.appengine.api.datastore.Key.class);
@@ -0,0 +1,90 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.ImmutableObject;
+import google.registry.model.registry.label.PremiumList;
+import google.registry.testing.AppEngineExtension;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit tests for {@link PremiumListKeyConverter}. */
+class PremiumListKeyConverterTest {
+
+  @RegisterExtension
+  final AppEngineExtension appEngine =
+      AppEngineExtension.builder()
+          .withDatastoreAndCloudSql()
+          .withJpaUnitTestEntities(PremiumListEntity.class)
+          .build();
+
+  private final PremiumListKeyConverter converter = new PremiumListKeyConverter();
+
+  @Test
+  void convertToDatabaseColumn_returnsNullIfInputIsNull() {
+    assertThat(converter.convertToDatabaseColumn(null)).isNull();
+  }
+
+  @Test
+  void convertToDatabaseColumn_convertsCorrectly() {
+    assertThat(
+            converter.convertToDatabaseColumn(
+                Key.create(getCrossTldKey(), PremiumList.class, "testList")))
+        .isEqualTo("testList");
+  }
+
+  @Test
+  void convertToEntityAttribute_returnsNullIfInputIsNull() {
+    assertThat(converter.convertToEntityAttribute(null)).isNull();
+  }
+
+  @Test
+  void convertToEntityAttribute_convertsCorrectly() {
+    assertThat(converter.convertToEntityAttribute("testList"))
+        .isEqualTo(Key.create(getCrossTldKey(), PremiumList.class, "testList"));
+  }
+
+  @Test
+  void testRoundTrip() {
+    Key<PremiumList> key = Key.create(getCrossTldKey(), PremiumList.class, "test");
+    PremiumListEntity testEntity = new PremiumListEntity(key);
+    jpaTm().transact(() -> jpaTm().getEntityManager().persist(testEntity));
+    PremiumListEntity persisted =
+        jpaTm().transact(() -> jpaTm().getEntityManager().find(PremiumListEntity.class, "test"));
+    assertThat(persisted.premiumList).isEqualTo(key);
+  }
+
+  @Entity(name = "PremiumListEntity")
+  private static class PremiumListEntity extends ImmutableObject {
+
+    @Id String name;
+
+    Key<PremiumList> premiumList;
+
+    public PremiumListEntity() {}
+
+    PremiumListEntity(Key<PremiumList> premiumList) {
+      this.name = premiumList.getName();
+      this.premiumList = premiumList;
+    }
+  }
+}
@@ -0,0 +1,90 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.ImmutableObject;
+import google.registry.model.registry.label.ReservedList;
+import google.registry.testing.AppEngineExtension;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit tests for {@link ReservedListKeyConverter}. */
+class ReservedListKeyConverterTest {
+
+  @RegisterExtension
+  final AppEngineExtension appEngine =
+      AppEngineExtension.builder()
+          .withDatastoreAndCloudSql()
+          .withJpaUnitTestEntities(ReservedListEntity.class)
+          .build();
+
+  private final ReservedListKeyConverter converter = new ReservedListKeyConverter();
+
+  @Test
+  void convertToDatabaseColumn_returnsNullIfInputIsNull() {
+    assertThat(converter.convertToDatabaseColumn(null)).isNull();
+  }
+
+  @Test
+  void convertToDatabaseColumn_convertsCorrectly() {
+    assertThat(
+            converter.convertToDatabaseColumn(
+                Key.create(getCrossTldKey(), ReservedList.class, "testList")))
+        .isEqualTo("testList");
+  }
+
+  @Test
+  void convertToEntityAttribute_returnsNullIfInputIsNull() {
+    assertThat(converter.convertToEntityAttribute(null)).isNull();
+  }
+
+  @Test
+  void convertToEntityAttribute_convertsCorrectly() {
+    assertThat(converter.convertToEntityAttribute("testList"))
+        .isEqualTo(Key.create(getCrossTldKey(), ReservedList.class, "testList"));
+  }
+
+  @Test
+  void testRoundTrip() {
+    Key<ReservedList> key = Key.create(getCrossTldKey(), ReservedList.class, "test");
+    ReservedListEntity testEntity = new ReservedListEntity(key);
+    jpaTm().transact(() -> jpaTm().getEntityManager().persist(testEntity));
+    ReservedListEntity persisted =
+        jpaTm().transact(() -> jpaTm().getEntityManager().find(ReservedListEntity.class, "test"));
+    assertThat(persisted.reservedList).isEqualTo(key);
+  }
+
+  @Entity(name = "ReservedListEntity")
+  private static class ReservedListEntity extends ImmutableObject {
+
+    @Id String name;
+
+    Key<ReservedList> reservedList;
+
+    public ReservedListEntity() {}
+
+    ReservedListEntity(Key<ReservedList> reservedList) {
+      this.name = reservedList.getName();
+      this.reservedList = reservedList;
+    }
+  }
+}
@@ -20,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;

 import google.registry.model.ImmutableObject;
 import google.registry.persistence.transaction.JpaTestRules.JpaUnitTestExtension;
-import google.registry.schema.tmch.ClaimsList;
 import java.util.List;
 import javax.persistence.Entity;
 import javax.persistence.Id;
@@ -36,9 +35,7 @@ public class JpaTransactionManagerRuleTest {

  @RegisterExtension
  public final JpaUnitTestExtension jpaExtension =
-      new JpaTestRules.Builder()
-          .withEntityClass(ClaimsList.class, TestEntity.class)
-          .buildUnitTestRule();
+      new JpaTestRules.Builder().withEntityClass(TestEntity.class).buildUnitTestRule();

  @Test
  void verifiesRuleWorks() {
@@ -58,7 +55,7 @@ public class JpaTransactionManagerRuleTest {
              List results =
                  jpaTm()
                      .getEntityManager()
-                      .createNativeQuery("SELECT * FROM \"ClaimsList\"")
+                      .createNativeQuery("SELECT * FROM \"TestEntity\"")
                      .getResultList();
              assertThat(results).isEmpty();
            });
@@ -37,6 +37,8 @@ import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.stream.Stream;
+import javax.persistence.Embeddable;
+import javax.persistence.MappedSuperclass;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.TestTemplate;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -65,7 +67,7 @@ public class TransactionManagerTest {
          .withClock(fakeClock)
          .withDatastoreAndCloudSql()
          .withOfyTestEntities(TestEntity.class)
-          .withJpaUnitTestEntities(TestEntity.class)
+          .withJpaUnitTestEntities(TestEntity.class, TestEntityBase.class)
          .build();

  TransactionManagerTest() {}
@@ -324,17 +326,31 @@ public class TransactionManagerTest {
    entities.forEach(TransactionManagerTest::assertEntityNotExist);
  }

+  /**
+   * We put the id field into a base class to test that id fields can be discovered in a base class.
+   */
+  @MappedSuperclass
+  @Embeddable
+  private static class TestEntityBase extends ImmutableObject {
+    @Id @javax.persistence.Id protected String name;
+
+    TestEntityBase(String name) {
+      this.name = name;
+    }
+
+    TestEntityBase() {}
+  }
+
  @Entity(name = "TxnMgrTestEntity")
  @javax.persistence.Entity(name = "TestEntity")
-  private static class TestEntity extends ImmutableObject {
-    @Id @javax.persistence.Id private String name;
+  private static class TestEntity extends TestEntityBase {

    private String data;

    private TestEntity() {}

    private TestEntity(String name, String data) {
-      this.name = name;
+      super(name);
      this.data = data;
    }

@@ -21,6 +21,8 @@ import static google.registry.testing.DatastoreHelper.createTld;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assertions.assertThrows;

+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
 import com.google.api.client.http.LowLevelHttpRequest;
 import com.google.api.client.http.LowLevelHttpResponse;
 import com.google.api.client.testing.http.MockHttpTransport;
@@ -50,7 +52,8 @@ class IcannHttpReporterTest {
  AppEngineExtension appEngineRule =
      new AppEngineExtension.Builder().withDatastoreAndCloudSql().build();

-  private MockHttpTransport createMockTransport(final ByteSource iirdeaResponse) {
+  private MockHttpTransport createMockTransport(
+      int statusCode, final ByteSource iirdeaResponse) {
    return new MockHttpTransport() {
      @Override
      public LowLevelHttpRequest buildRequest(String method, String url) {
@@ -59,7 +62,7 @@ class IcannHttpReporterTest {
              @Override
              public LowLevelHttpResponse execute() throws IOException {
                MockLowLevelHttpResponse response = new MockLowLevelHttpResponse();
-                response.setStatusCode(200);
+                response.setStatusCode(statusCode);
                response.setContentType(PLAIN_TEXT_UTF_8.toString());
                response.setContent(iirdeaResponse.read());
                return response;
@@ -71,6 +74,10 @@ class IcannHttpReporterTest {
    };
  }

+  private MockHttpTransport createMockTransport(final ByteSource iirdeaResponse) {
+    return createMockTransport(HttpStatusCodes.STATUS_CODE_OK, iirdeaResponse);
+  }
+
  @BeforeEach
  void beforeEach() {
    createTld("test");
@@ -117,10 +124,21 @@ class IcannHttpReporterTest {
  @Test
  void testFail_BadIirdeaResponse() throws Exception {
    IcannHttpReporter reporter = createReporter();
-    reporter.httpTransport = createMockTransport(IIRDEA_BAD_XML);
+    reporter.httpTransport =
+        createMockTransport(HttpStatusCodes.STATUS_CODE_BAD_REQUEST, IIRDEA_BAD_XML);
    assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse();
  }

+  @Test
+  void testFail_transportException() throws Exception {
+    IcannHttpReporter reporter = createReporter();
+    reporter.httpTransport =
+        createMockTransport(HttpStatusCodes.STATUS_CODE_FORBIDDEN, ByteSource.empty());
+    assertThrows(
+        HttpResponseException.class,
+        () -> reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv"));
+  }
+
  @Test
  void testFail_invalidFilename_nonSixDigitYearMonth() {
    IcannHttpReporter reporter = createReporter();
@@ -27,6 +27,7 @@ import google.registry.model.poll.PollMessageTest;
 import google.registry.model.registry.RegistryLockDaoTest;
 import google.registry.model.registry.label.ReservedListSqlDaoTest;
 import google.registry.model.reporting.Spec11ThreatMatchTest;
+import google.registry.model.tmch.ClaimsListDaoTest;
 import google.registry.persistence.transaction.JpaEntityCoverageExtension;
 import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
 import google.registry.schema.cursor.CursorDaoTest;
@@ -35,7 +36,6 @@ import google.registry.schema.integration.SqlIntegrationTestSuite.BeforeSuiteTes
 import google.registry.schema.registrar.RegistrarDaoTest;
 import google.registry.schema.server.LockDaoTest;
 import google.registry.schema.tld.PremiumListDaoTest;
-import google.registry.schema.tmch.ClaimsListDaoTest;
 import google.registry.testing.AppEngineExtension;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -41,24 +41,32 @@ public class EntityTest {
        new ClassGraph().enableAnnotationInfo().whitelistPackages("google.registry").scan()) {
      // All javax.persistence entities must implement SqlEntity and vice versa
      ImmutableSet<String> javaxPersistenceClasses =
-          getClassNames(
-              scanResult.getClassesWithAnnotation(javax.persistence.Entity.class.getName()));
+          getAllClassesWithAnnotation(scanResult, javax.persistence.Entity.class.getName());
      ImmutableSet<String> sqlEntityClasses =
          getClassNames(scanResult.getClassesImplementing(SqlEntity.class.getName()));
-      assertThat(javaxPersistenceClasses).isEqualTo(sqlEntityClasses);
+      assertThat(sqlEntityClasses).containsExactlyElementsIn(javaxPersistenceClasses);

-      // All com.googlecode.objectify.annotation.Entity classes must implement DatastoreEntity and
-      // vice versa
+      // All com.googlecode.objectify entities must implement DatastoreEntity and vice versa
      ImmutableSet<String> objectifyClasses =
-          getClassNames(
-              scanResult.getClassesWithAnnotation(
-                  com.googlecode.objectify.annotation.Entity.class.getName()));
+          getAllClassesWithAnnotation(
+              scanResult, com.googlecode.objectify.annotation.Entity.class.getName());
      ImmutableSet<String> datastoreEntityClasses =
          getClassNames(scanResult.getClassesImplementing(DatastoreEntity.class.getName()));
-      assertThat(objectifyClasses).isEqualTo(datastoreEntityClasses);
+      assertThat(datastoreEntityClasses).containsExactlyElementsIn(objectifyClasses);
    }
  }

+  private ImmutableSet<String> getAllClassesWithAnnotation(
+      ScanResult scanResult, String annotation) {
+    ImmutableSet.Builder<String> result = new ImmutableSet.Builder<>();
+    ClassInfoList classesWithAnnotation = scanResult.getClassesWithAnnotation(annotation);
+    result.addAll(getClassNames(classesWithAnnotation));
+    classesWithAnnotation.stream()
+        .map(ClassInfo::getSubclasses)
+        .forEach(classInfoList -> result.addAll(getClassNames(classInfoList)));
+    return result.build();
+  }
+
  private ImmutableSet<String> getClassNames(ClassInfoList classInfoList) {
    return classInfoList.stream()
        .filter(ClassInfo::isStandardClass)
@@ -23,11 +23,11 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.VirtualEntity;
 import google.registry.model.common.EntityGroupRoot;
+import google.registry.schema.replay.EntityTest.EntityForTesting;

-/**
- * A test model object that can be persisted in any entity group.
- */
+/** A test model object that can be persisted in any entity group. */
@Entity
+@EntityForTesting
 public class TestObject extends ImmutableObject {

  @Parent
@@ -65,6 +65,7 @@ public class TestObject extends ImmutableObject {
  /** A test @VirtualEntity model object, which should not be persisted. */
  @Entity
  @VirtualEntity
+  @EntityForTesting
  public static class TestVirtualObject extends ImmutableObject {

    @Id
@@ -50,7 +50,8 @@ class TmchDnlActionTest extends TmchActionTestCase {

    // Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
    ClaimsListShard claimsList = ClaimsListShard.get();
-    assertThat(claimsList.getCreationTime()).isEqualTo(DateTime.parse("2013-11-24T23:15:37.4Z"));
+    assertThat(claimsList.getTmdbGenerationTime())
+        .isEqualTo(DateTime.parse("2013-11-24T23:15:37.4Z"));
    assertThat(claimsList.getClaimKey("xn----7sbejwbn3axu3d"))
        .hasValue("2013112500/7/4/8/dIHW0DiuybvhdP8kIz");
    assertThat(claimsList.getClaimKey("lolcat")).isEmpty();
@@ -37,7 +37,8 @@ class UploadClaimsListCommandTest extends CommandTestCase<UploadClaimsListComman
    runCommand("--force", filename);

    ClaimsListShard claimsList = ClaimsListShard.get();
-    assertThat(claimsList.getCreationTime()).isEqualTo(DateTime.parse("2012-08-16T00:00:00.0Z"));
+    assertThat(claimsList.getTmdbGenerationTime())
+        .isEqualTo(DateTime.parse("2012-08-16T00:00:00.0Z"));
    assertThat(claimsList.getClaimKey("example"))
        .hasValue("2013041500/2/6/9/rJ1NrDO92vDsAzf7EQzgjX4R0000000001");
    assertThat(claimsList.getClaimKey("another-example"))
@@ -55,9 +55,7 @@ FROM (
    FROM
      `my-project-id.latest_datastore_export.Registry`
    WHERE
-    -- TODO(b/18092292): Add a filter for tldState (not PDT/PREDELEGATION)
-      tldType = 'REAL'
-    AND disableInvoicing is not TRUE) ) AS BillingEvent
+      enableInvoicing IS TRUE) ) AS BillingEvent
  -- Gather billing ID from registrar table
  -- This is a 'JOIN' as opposed to 'LEFT JOIN' to filter out
  -- non-billable registrars
@@ -270,9 +270,9 @@ class google.registry.model.domain.DomainHistory {
  google.registry.model.domain.Period period;
  google.registry.model.eppcommon.Trid trid;
  google.registry.model.reporting.HistoryEntry$Type type;
-  google.registry.persistence.VKey<google.registry.model.domain.DomainBase> domainRepoId;
  java.lang.Boolean requestedByRegistrar;
  java.lang.String clientId;
+  java.lang.String domainRepoId;
  java.lang.String otherClientId;
  java.lang.String reason;
  java.util.Set<google.registry.model.reporting.DomainTransactionRecord> domainTransactionRecords;
@@ -618,9 +618,9 @@ enum google.registry.model.registrar.RegistrarContact$Type {
 class google.registry.model.registry.Registry {
  @Id java.lang.String tldStrId;
  @Parent com.googlecode.objectify.Key<google.registry.model.common.EntityGroupRoot> parent;
-  boolean disableInvoicing;
  boolean dnsPaused;
  boolean escrowEnabled;
+  boolean invoicingEnabled;
  com.googlecode.objectify.Key<google.registry.model.registry.label.PremiumList> premiumList;
  google.registry.model.CreateAutoTimestamp creationTime;
  google.registry.model.common.TimedTransitionProperty<google.registry.model.registry.Registry$TldState, google.registry.model.registry.Registry$TldStateTransition> tldStateTransitions;
@@ -0,0 +1,34 @@
+-- Copyright 2020 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter sequence "history_id_sequence" increment 50;
+
+alter table "DomainHistory" alter column history_revision_id drop default;
+alter table "ContactHistory" alter column history_revision_id drop default;
+alter table "HostHistory" alter column history_revision_id drop default;
+
+alter table if exists "DomainHistoryHost"
+    drop constraint fk6b8eqdxwe3guc56tgpm89atx;
+
+alter table "DomainHistory" drop constraint "DomainHistory_pkey";
+
+alter table "DomainHistory"
+    add constraint "DomainHistory_pkey" primary key (domain_repo_id, history_revision_id);
+
+alter table "DomainHistoryHost" add column domain_history_domain_repo_id text not null;
+
+alter table if exists "DomainHistoryHost"
+    add constraint FKa9woh3hu8gx5x0vly6bai327n
+    foreign key (domain_history_domain_repo_id, domain_history_history_revision_id)
+    references "DomainHistory";
@@ -0,0 +1,19 @@
+-- Copyright 2020 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+ALTER TABLE ONLY public."Domain"
+    DROP CONSTRAINT fk_domain_billing_recurrence_id;
+ALTER TABLE ONLY public."Domain"
+    ADD CONSTRAINT fk_domain_billing_recurrence_id FOREIGN KEY (billing_recurrence_id)
+    REFERENCES public."BillingRecurrence"(billing_recurrence_id);
@@ -0,0 +1,20 @@
+-- Copyright 2020 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+create sequence "temp_history_id_sequence"
+    start with 1
+    increment by 50
+    no minvalue
+    no maxvalue
+    cache 1;
@@ -11,7 +11,7 @@
 -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
-create sequence history_id_sequence start 1 increment 1;
+create sequence temp_history_id_sequence start 1 increment 50;

    create table "AllocationToken" (
       token text not null,
@@ -86,8 +86,8 @@ create sequence history_id_sequence start 1 increment 1;

    create table "ClaimsList" (
       revision_id  bigserial not null,
-        creation_timestamp timestamptz not null,
        tmdb_generation_time timestamptz not null,
+        creation_timestamp timestamptz not null,
        primary key (revision_id)
    );

@@ -286,7 +286,8 @@ create sequence history_id_sequence start 1 increment 1;
    );

    create table "DomainHistory" (
-       history_revision_id int8 not null,
+       domain_repo_id text not null,
+        history_revision_id int8 not null,
        history_by_superuser boolean not null,
        history_registrar_id text,
        history_modification_time timestamptz not null,
@@ -341,12 +342,12 @@ create sequence history_id_sequence start 1 increment 1;
        last_epp_update_time timestamptz,
        statuses text[],
        update_timestamp timestamptz,
-        domain_repo_id text not null,
-        primary key (history_revision_id)
+        primary key (domain_repo_id, history_revision_id)
    );

    create table "DomainHistoryHost" (
-       domain_history_history_revision_id int8 not null,
+       domain_history_domain_repo_id text not null,
+        domain_history_history_revision_id int8 not null,
        host_repo_id text
    );

@@ -644,8 +645,8 @@ create index spec11threatmatch_check_date_idx on "Spec11ThreatMatch" (check_date
       references "ClaimsList";

    alter table if exists "DomainHistoryHost" 
-       add constraint FK378h8v3j8qd8xtjn2e0bcmrtj 
-       foreign key (domain_history_history_revision_id) 
+       add constraint FKa9woh3hu8gx5x0vly6bai327n 
+       foreign key (domain_history_domain_repo_id, domain_history_history_revision_id) 
       references "DomainHistory";

    alter table if exists "DomainHost" 
@@ -275,24 +275,12 @@ CREATE TABLE public."Contact" (
 );


--
-- Name: history_id_sequence; Type: SEQUENCE; Schema: public; Owner: -
--
-
-CREATE SEQUENCE public.history_id_sequence
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
 --
 -- Name: ContactHistory; Type: TABLE; Schema: public; Owner: -
 --

 CREATE TABLE public."ContactHistory" (
-    history_revision_id bigint DEFAULT nextval('public.history_id_sequence'::regclass) NOT NULL,
+    history_revision_id bigint NOT NULL,
    history_by_superuser boolean NOT NULL,
    history_registrar_id text,
    history_modification_time timestamp with time zone NOT NULL,
@@ -431,7 +419,7 @@ CREATE TABLE public."Domain" (
 --

 CREATE TABLE public."DomainHistory" (
-    history_revision_id bigint DEFAULT nextval('public.history_id_sequence'::regclass) NOT NULL,
+    history_revision_id bigint NOT NULL,
    history_by_superuser boolean NOT NULL,
    history_registrar_id text,
    history_modification_time timestamp with time zone NOT NULL,
@@ -496,7 +484,8 @@ CREATE TABLE public."DomainHistory" (

 CREATE TABLE public."DomainHistoryHost" (
    domain_history_history_revision_id bigint NOT NULL,
-    host_repo_id text
+    host_repo_id text,
+    domain_history_domain_repo_id text NOT NULL
 );


@@ -549,7 +538,7 @@ ALTER SEQUENCE public."GracePeriod_id_seq" OWNED BY public."GracePeriod".id;
 --

 CREATE TABLE public."HostHistory" (
-    history_revision_id bigint DEFAULT nextval('public.history_id_sequence'::regclass) NOT NULL,
+    history_revision_id bigint NOT NULL,
    history_by_superuser boolean NOT NULL,
    history_registrar_id text NOT NULL,
    history_modification_time timestamp with time zone NOT NULL,
@@ -929,6 +918,30 @@ CREATE SEQUENCE public."Transaction_id_seq"
 ALTER SEQUENCE public."Transaction_id_seq" OWNED BY public."Transaction".id;


+--
+-- Name: history_id_sequence; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.history_id_sequence
+    START WITH 1
+    INCREMENT BY 50
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: temp_history_id_sequence; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.temp_history_id_sequence
+    START WITH 1
+    INCREMENT BY 50
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
 --
 -- Name: BillingCancellation billing_cancellation_id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -1083,7 +1096,7 @@ ALTER TABLE ONLY public."Cursor"
 --

 ALTER TABLE ONLY public."DomainHistory"
-    ADD CONSTRAINT "DomainHistory_pkey" PRIMARY KEY (history_revision_id);
+    ADD CONSTRAINT "DomainHistory_pkey" PRIMARY KEY (domain_repo_id, history_revision_id);


 --
@@ -1612,14 +1625,6 @@ ALTER TABLE ONLY public."HostHistory"
    ADD CONSTRAINT fk3d09knnmxrt6iniwnp8j2ykga FOREIGN KEY (history_registrar_id) REFERENCES public."Registrar"(registrar_id);


--
-- Name: DomainHistoryHost fk6b8eqdxwe3guc56tgpm89atx; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public."DomainHistoryHost"
-    ADD CONSTRAINT fk6b8eqdxwe3guc56tgpm89atx FOREIGN KEY (domain_history_history_revision_id) REFERENCES public."DomainHistory"(history_revision_id);
-
-
 --
 -- Name: ClaimsEntry fk6sc6at5hedffc0nhdcab6ivuq; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -1745,7 +1750,7 @@ ALTER TABLE ONLY public."Domain"
 --

 ALTER TABLE ONLY public."Domain"
-    ADD CONSTRAINT fk_domain_billing_recurrence_id FOREIGN KEY (billing_recurrence_id) REFERENCES public."BillingEvent"(billing_event_id);
+    ADD CONSTRAINT fk_domain_billing_recurrence_id FOREIGN KEY (billing_recurrence_id) REFERENCES public."BillingRecurrence"(billing_recurrence_id);


 --
@@ -1916,6 +1921,14 @@ ALTER TABLE ONLY public."PollMessage"
    ADD CONSTRAINT fk_poll_message_transfer_response_losing_registrar_id FOREIGN KEY (transfer_response_losing_registrar_id) REFERENCES public."Registrar"(registrar_id);


+--
+-- Name: DomainHistoryHost fka9woh3hu8gx5x0vly6bai327n; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."DomainHistoryHost"
+    ADD CONSTRAINT fka9woh3hu8gx5x0vly6bai327n FOREIGN KEY (domain_history_domain_repo_id, domain_history_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+
+
 --
 -- Name: DomainHost fkfmi7bdink53swivs390m2btxg; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -131,7 +131,7 @@ ext {
      'org.bouncycastle:bcpg-jdk15on:1.61',
      'org.bouncycastle:bcpkix-jdk15on:1.61',
      'org.bouncycastle:bcprov-jdk15on:1.61',
-      'com.fasterxml.jackson.core:jackson-databind:2.9.10',
+      'com.fasterxml.jackson.core:jackson-databind:2.11.2',
      'org.flywaydb:flyway-core:5.2.4',
      'org.glassfish.jaxb:jaxb-runtime:2.3.0',
      'org.hamcrest:hamcrest-all:1.3',
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
@@ -130,7 +130,7 @@ fi
 if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    
+
    JAVACMD=`cygpath --unix "$JAVACMD"`

    # We build the pattern for arguments to be converted via cygpath
@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome

 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
+if "%ERRORLEVEL%" == "0" goto execute

 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -54,7 +54,7 @@ goto fail
 set JAVA_HOME=%JAVA_HOME:"=%
 set JAVA_EXE=%JAVA_HOME%/bin/java.exe

-if exist "%JAVA_EXE%" goto init
+if exist "%JAVA_EXE%" goto execute

 echo.
 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
@@ -64,21 +64,6 @@ echo location of your Java installation.

 goto fail

-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
 :execute
@rem Setup the command line

@@ -86,7 +71,7 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar


@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*

 :end
@rem End local scope for the variables with windows NT shell
Author	SHA1	Message	Date
Michael Muller	5488e1b323	Fix accessing superclass fields in checkExists() (#799 ) * Fix accessing superclass fields in checkExists() JpaTransactionManagerImpl doesn't respect @Id fields in mapped superclasses. Replace calls to getDeclaredId() and getDeclaredField() with superclass friendly counterparts.	2020-09-11 13:45:51 -04:00
Shicong Huang	5ab0f97351	Add and use temp_history_id_sequence to avoid release error (#795 )	2020-09-11 12:25:08 -04:00
sarahcaseybot	f7b65327da	Add type converter for Key<ReservedList> and Key<PremiumList> (#796 ) * Add converter for reservedlist and premiumlist keys * Remove public modifier from test classes	2020-09-10 17:36:22 -04:00
Michael Muller	36482ce94f	Fix the billing occurrence foreign key (#797 ) * Fix the billing occurrence foreign key Fix the Domain.billing_occurrence_id foreign key constraint to reference the correct table (BillingRecurrence, not BillingEvent).	2020-09-10 12:02:24 -04:00
Lai Jiang	125f509b46	Change disable invoicing flag to enable invoicing flag (#783 ) * Change disable invoicing flag to enable invoicing flag This flag will be the sole determinor on if invoicing is enabled, regardless of TLD types. Once this PR is deployed we will need to run the nomulus command to update this flag on all launched open TLDs. For context on why this change is made, see b/159626744. * Rename enableInvoicing to InvoicingEnabled	2020-09-09 19:37:41 -04:00
gbrodman	fb7ba80b86	Implement DatastoreEntity/SqlEntity for many more classes (#788 ) * Implement DatastoreEntity/SqlEntity for many more classes We still have many more classes to go, but this gets us closer to guaranteeing that we can convert from Datastore to SQL objects and back again. * Shift SqlEntity impl to HistoryEntry	2020-09-09 13:56:59 -04:00
Lai Jiang	a86fcf79f7	Make ICANN reporting not fail on success upload (#791 ) * Make ICANN reporting not fail on success upload According to the spec (https://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-13#page-16), when an upload succeeds (HTTP response code 200), the result code contained in the response message is always 1000 (success). So there is no need to parse the response content and check the result code. Given that we are having a problem parsing the response content due to encoding, it is best that we don't check it so as to not get false negative alerts when the upload is successful. The current logic also has a bug: HttpRequest.execute() will by default throw when the response code is non-20X. Therefore for a 400 response, our parsing logic never runs on it. Coincidentally, this month when we uploaded the July activity report (due to stale cursors), we get 400 responses (due to existing reports on the ICANN servers). The stack trace printed for the thrown exceptions from the 400 responses contained correctly parsed response contents. This lead us to believe that the issue with encoding was transient last month. However when we tried again to upload this month's report, our parser failed again (because the response code was 200 this time, and our parser actually ran on the response contents). This seems to suggest that ICANN is sending back readable response contents, but our parser somehow failed to understand it, assuming that ICANN is using the same encoding for 200 (which we tried and failed to parse) and 400 response contents (which caused an exception and was printed corrected in the stack trace). This PR changed the transport behavior so that it doesn't throw automatically for non-20X responses. We will print the content for both 200 and 400 responses, but only try to parse 400 response content. We put the 400 response in an HttpResponseException and print stack trace from it, which should display the content correctly so that we can compare it with the result of our own parsing. * Add tests	2020-09-03 15:57:30 -04:00
Lai Jiang	dc8e095e55	Upgrade to Gradle 6.6.1 (#792 )	2020-09-03 15:56:52 -04:00
Shicong Huang	cdf2c7f7cb	Merge ClaimsList into ClaimsListShard (#694 ) * Merge ClaimsList into ClaimsListShard * Add a TODO to rename the class * Rebase on HEAD * Improve javadoc	2020-09-03 11:18:40 -04:00
Shicong Huang	ecafebdc3d	Use composite primary key for DomainHistory (#767 ) * Use composite primary key for DomainHistory * Move History table's SequenceGenerator to orm.xml * Rebase on HEAD and remove default value for key in History tables * Use primitive type for id. * Revert the cache change	2020-09-03 10:21:23 -04:00
Lai Jiang	c6c8d21281	Update jackson-core to the latest version (#789 ) Vomit identified a vulnerability in the current version.	2020-09-03 09:11:12 -04:00