mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-06-08 16:02:56 +00:00
Code Issues Releases Wiki Activity

Remove bulk query entities (#1834)

Browse Source 
These alternative ORMs are introduced as a way to make querying large number of
domains and domain histories more efficient through bulk loading from several
to-be-joined tables separately, then in-memory re-assembly of the final entity,
bypassing the need to query multiple tables each time an entity is queried.

Their primary use case is loading these entities for comparison between
datastore and SQL during the migration, which has been completed. The
code remain unused as of now and their existence makes refactoring and
general maintenance more complicated than necessary due to the need to keep
them up to date.

Therefore we remove the related code.

<!-- Reviewable:start -->
- - -
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1834)
<!-- Reviewable:end -->
This commit is contained in:
Lai Jiang
2022-10-28 12:25:57 -04:00
committed by GitHub
parent 3f68ad5ea3
commit 1c90a6648e
15 changed files with 0 additions and 1088 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java
View File

@@ -20,9 +20,7 @@ import dagger.Lazy;
import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.model.domain.Domain;
import google.registry.persistence.PersistenceModule;
import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
import google.registry.persistence.PersistenceModule.BeamJpaTm;
import google.registry.persistence.PersistenceModule.BeamReadOnlyReplicaJpaTm;
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
@@ -52,14 +50,6 @@ public interface RegistryPipelineComponent {
@BeamJpaTm
Lazy<JpaTransactionManager> getJpaTransactionManager();
/**
* Returns a {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities
* ({@link Domain} and {@link google.registry.model.domain.DomainHistory}). Please refer to {@link
* google.registry.model.bulkquery.BulkQueryEntities} for more information.
*/
@BeamBulkQueryJpaTm
Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
/**
* A {@link JpaTransactionManager} that uses the Postgres read-only replica if configured (uses
* the standard DB otherwise).

3
core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java
View File

@@ -55,9 +55,6 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
toRegistryPipelineComponent(registryOptions);
Lazy<JpaTransactionManager> transactionManagerLazy;
switch (registryOptions.getJpaTransactionManagerType()) {
case BULK_QUERY:
transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
break;
case READ_ONLY_REPLICA:
transactionManagerLazy =
registryPipelineComponent.getReadOnlyReplicaJpaTransactionManager();

112
core/src/main/java/google/registry/model/bulkquery/BulkQueryEntities.java
View File

@@ -1,112 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import google.registry.model.domain.Domain;
import google.registry.model.domain.DomainBase;
import google.registry.model.domain.DomainHistory;
import google.registry.model.domain.GracePeriod;
import google.registry.model.domain.GracePeriod.GracePeriodHistory;
import google.registry.model.domain.secdns.DomainDsData;
import google.registry.model.domain.secdns.DomainDsDataHistory;
import google.registry.model.host.Host;
import google.registry.model.reporting.DomainTransactionRecord;
import google.registry.persistence.VKey;
import google.registry.persistence.transaction.JpaTransactionManager;
/**
* Utilities for managing an alternative JPA entity model optimized for bulk loading multi-level
* entities such as {@link Domain} and {@link DomainHistory}.
*
* <p>In a bulk query for a multi-level JPA entity type, the JPA framework only generates a bulk
* query (SELECT * FROM table) for the base table. Then, for each row in the base table, additional
* queries are issued to load associated rows in child tables. This can be very slow when an entity
* type has multiple child tables.
*
* <p>We have defined an alternative entity model for {@link Domain} and {@link DomainHistory},
* where the base table as well as the child tables are mapped to single-level entity types. The
* idea is to load each of these types using a bulk query, and assemble them into the target type in
* memory in a pipeline. The main use case is Datastore-Cloud SQL validation during the Registry
* database migration, where we will need the full database snapshots frequently.
*/
public class BulkQueryEntities {
/**
* The JPA entity classes in persistence.xml to replace when creating the {@link
* JpaTransactionManager} for bulk query.
*/
public static final ImmutableMap<String, String> JPA_ENTITIES_REPLACEMENTS =
ImmutableMap.of(
Domain.class.getCanonicalName(),
DomainLite.class.getCanonicalName(),
DomainHistory.class.getCanonicalName(),
DomainHistoryLite.class.getCanonicalName());
/* The JPA entity classes that are not included in persistence.xml and need to be added to
* the {@link JpaTransactionManager} for bulk query.*/
public static final ImmutableList<String> JPA_ENTITIES_NEW =
ImmutableList.of(
DomainHost.class.getCanonicalName(), DomainHistoryHost.class.getCanonicalName());
public static Domain assembleDomain(
DomainLite domainLite,
ImmutableSet<GracePeriod> gracePeriods,
ImmutableSet<DomainDsData> domainDsData,
ImmutableSet<VKey<Host>> nsHosts) {
Domain.Builder builder = new Domain.Builder();
builder.copyFrom(domainLite);
builder.setGracePeriods(gracePeriods);
builder.setDsData(domainDsData);
builder.setNameservers(nsHosts);
// Restore the original update timestamp (this gets cleared when we set nameservers or DS data).
builder.setUpdateTimestamp(domainLite.getUpdateTimestamp());
return builder.build();
}
public static DomainHistory assembleDomainHistory(
DomainHistoryLite domainHistoryLite,
ImmutableSet<DomainDsDataHistory> dsDataHistories,
ImmutableSet<VKey<Host>> domainHistoryHosts,
ImmutableSet<GracePeriodHistory> gracePeriodHistories,
ImmutableSet<DomainTransactionRecord> transactionRecords) {
DomainHistory.Builder builder = new DomainHistory.Builder();
builder.copyFrom(domainHistoryLite);
DomainBase rawDomainBase = domainHistoryLite.domainBase;
if (rawDomainBase != null) {
DomainBase newDomainBase =
domainHistoryLite
.domainBase
.asBuilder()
.setNameservers(domainHistoryHosts)
.setGracePeriods(
gracePeriodHistories.stream()
.map(GracePeriod::createFromHistory)
.collect(toImmutableSet()))
.setDsData(
dsDataHistories.stream().map(DomainDsData::create).collect(toImmutableSet()))
// Restore the original update timestamp (this gets cleared when we set nameservers or
// DS data).
.setUpdateTimestamp(domainHistoryLite.domainBase.getUpdateTimestamp())
.build();
builder.setDomain(newDomainBase);
}
return builder.buildAndAssemble(
dsDataHistories, domainHistoryHosts, gracePeriodHistories, transactionRecords);
}
}

69
core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java
View File

@@ -1,69 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import com.google.common.base.Objects;
import google.registry.model.domain.DomainHistory.DomainHistoryId;
import google.registry.model.host.Host;
import google.registry.persistence.VKey;
import java.io.Serializable;
import javax.persistence.Access;
import javax.persistence.AccessType;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.IdClass;
/**
* A name server host referenced by a {@link google.registry.model.domain.DomainHistory} record.
* Please refer to {@link BulkQueryEntities} for usage.
*/
@Entity
@Access(AccessType.FIELD)
@IdClass(DomainHistoryHost.class)
public class DomainHistoryHost implements Serializable {
@Id private Long domainHistoryHistoryRevisionId;
@Id private String domainHistoryDomainRepoId;
@Id private String hostRepoId;
private DomainHistoryHost() {}
public DomainHistoryId getDomainHistoryId() {
return new DomainHistoryId(domainHistoryDomainRepoId, domainHistoryHistoryRevisionId);
}
public VKey<Host> getHostVKey() {
return VKey.create(Host.class, hostRepoId);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof DomainHistoryHost)) {
return false;
}
DomainHistoryHost that = (DomainHistoryHost) o;
return Objects.equal(domainHistoryHistoryRevisionId, that.domainHistoryHistoryRevisionId)
&& Objects.equal(domainHistoryDomainRepoId, that.domainHistoryDomainRepoId)
&& Objects.equal(hostRepoId, that.hostRepoId);
}
@Override
public int hashCode() {
return Objects.hashCode(domainHistoryHistoryRevisionId, domainHistoryDomainRepoId, hostRepoId);
}
}

125
core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java
View File

@@ -1,125 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import com.googlecode.objectify.Key;
import google.registry.model.domain.Domain;
import google.registry.model.domain.DomainBase;
import google.registry.model.domain.DomainHistory;
import google.registry.model.domain.DomainHistory.DomainHistoryId;
import google.registry.model.domain.Period;
import google.registry.model.reporting.HistoryEntry;
import google.registry.persistence.VKey;
import javax.annotation.Nullable;
import javax.persistence.Access;
import javax.persistence.AccessType;
import javax.persistence.AttributeOverride;
import javax.persistence.AttributeOverrides;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.IdClass;
import javax.persistence.PostLoad;
/**
* A 'light' version of {@link DomainHistory} with only base table ("DomainHistory") attributes,
* which allows fast bulk loading. They are used in in-memory assembly of {@code DomainHistory}
* instances along with bulk-loaded child entities ({@code GracePeriodHistory} etc). The in-memory
* assembly achieves much higher performance than loading {@code DomainHistory} directly.
*
* <p>Please refer to {@link BulkQueryEntities} for more information.
*
* <p>This class is adapted from {@link DomainHistory} by removing the {@code dsDataHistories},
* {@code gracePeriodHistories}, and {@code nsHosts} fields and associated methods.
*/
@Entity(name = "DomainHistory")
@Access(AccessType.FIELD)
@IdClass(DomainHistoryId.class)
public class DomainHistoryLite extends HistoryEntry {
// Store DomainBase instead of Domain so we don't pick up its @Id
// Nullable for the sake of pre-Registry-3.0 history objects
@Nullable DomainBase domainBase;
@Id
@Access(AccessType.PROPERTY)
public String getDomainRepoId() {
// We need to handle null case here because Hibernate sometimes accesses this method before
// parent gets initialized
return parent == null ? null : parent.getName();
}
/** This method is private because it is only used by Hibernate. */
@SuppressWarnings("unused")
private void setDomainRepoId(String domainRepoId) {
parent = Key.create(Domain.class, domainRepoId);
}
@Override
@Nullable
@Access(AccessType.PROPERTY)
@AttributeOverrides({
@AttributeOverride(name = "unit", column = @Column(name = "historyPeriodUnit")),
@AttributeOverride(name = "value", column = @Column(name = "historyPeriodValue"))
})
public Period getPeriod() {
return super.getPeriod();
}
/**
* For transfers, the id of the other registrar.
*
* <p>For requests and cancels, the other registrar is the losing party (because the registrar
* sending the EPP transfer command is the gaining party). For approves and rejects, the other
* registrar is the gaining party.
*/
@Nullable
@Access(AccessType.PROPERTY)
@Column(name = "historyOtherRegistrarId")
@Override
public String getOtherRegistrarId() {
return super.getOtherRegistrarId();
}
@Id
@Column(name = "historyRevisionId")
@Access(AccessType.PROPERTY)
@Override
public long getId() {
return super.getId();
}
/** The key to the {@link Domain} this is based off of. */
public VKey<Domain> getParentVKey() {
return VKey.create(Domain.class, getDomainRepoId());
}
public DomainHistoryId getDomainHistoryId() {
return new DomainHistoryId(getDomainRepoId(), getId());
}
@PostLoad
void postLoad() {
if (domainBase == null) {
return;
}
// See inline comments in DomainHistory.postLoad for reasons for the following lines.
if (domainBase.getDomainName() == null) {
domainBase = null;
} else if (domainBase.getRepoId() == null) {
domainBase.setRepoId(parent.getName());
}
}
}

64
core/src/main/java/google/registry/model/bulkquery/DomainHost.java
View File

@@ -1,64 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import com.google.common.base.Objects;
import google.registry.model.host.Host;
import google.registry.persistence.VKey;
import java.io.Serializable;
import javax.persistence.Access;
import javax.persistence.AccessType;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.IdClass;
/** A name server host of a domain. Please refer to {@link BulkQueryEntities} for usage. */
@Entity
@Access(AccessType.FIELD)
@IdClass(DomainHost.class)
public class DomainHost implements Serializable {
@Id private String domainRepoId;
@Id private String hostRepoId;
DomainHost() {}
public String getDomainRepoId() {
return domainRepoId;
}
public VKey<Host> getHostVKey() {
return VKey.create(Host.class, hostRepoId);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof DomainHost)) {
return false;
}
DomainHost that = (DomainHost) o;
return Objects.equal(domainRepoId, that.domainRepoId)
&& Objects.equal(hostRepoId, that.hostRepoId);
}
@Override
public int hashCode() {
return Objects.hashCode(domainRepoId, hostRepoId);
}
}

48
core/src/main/java/google/registry/model/bulkquery/DomainLite.java
View File

@@ -1,48 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import google.registry.model.domain.Domain;
import google.registry.model.domain.DomainBase;
import google.registry.persistence.VKey;
import google.registry.persistence.WithVKey;
import javax.persistence.Access;
import javax.persistence.AccessType;
import javax.persistence.Entity;
/**
* A 'light' version of {@link Domain} with only base table ("Domain") attributes, which allows fast
* bulk loading. They are used in in-memory assembly of {@code Domain} instances along with
* bulk-loaded child entities ({@code GracePeriod} etc). The in-memory assembly achieves much higher
* performance than loading {@code Domain} directly.
*
* <p>Please refer to {@link BulkQueryEntities} for more information.
*/
@Entity(name = "Domain")
@WithVKey(String.class)
@Access(AccessType.FIELD)
public class DomainLite extends DomainBase {
@Override
@javax.persistence.Id
@Access(AccessType.PROPERTY)
public String getRepoId() {
return super.getRepoId();
}
public static VKey<DomainLite> createVKey(String repoId) {
return VKey.createSql(DomainLite.class, repoId);
}
}

14
core/src/main/java/google/registry/model/domain/DomainHistory.java
View File

@@ -403,19 +403,5 @@ public class DomainHistory extends HistoryEntry {
fillAuxiliaryFieldsFromDomain(instance);
return instance;
}
public DomainHistory buildAndAssemble(
ImmutableSet<DomainDsDataHistory> dsDataHistories,
ImmutableSet<VKey<Host>> domainHistoryHosts,
ImmutableSet<GracePeriodHistory> gracePeriodHistories,
ImmutableSet<DomainTransactionRecord> transactionRecords) {
DomainHistory instance = super.build();
instance.dsDataHistories = dsDataHistories;
instance.nsHosts = domainHistoryHosts;
instance.gracePeriodHistories = gracePeriodHistories;
instance.domainTransactionRecords = transactionRecords;
instance.hashCode = null;
return instance;
}
}
}

65
core/src/main/java/google/registry/persistence/BulkQueryJpaFactory.java
View File

@@ -1,65 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.persistence;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import google.registry.model.bulkquery.BulkQueryEntities;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.persistence.transaction.JpaTransactionManagerImpl;
import google.registry.util.Clock;
import java.util.List;
import javax.persistence.EntityManagerFactory;
import org.hibernate.jpa.boot.internal.ParsedPersistenceXmlDescriptor;
import org.hibernate.jpa.boot.spi.Bootstrap;
/**
* Defines factory method for instantiating the bulk-query optimized {@link JpaTransactionManager}.
*/
public final class BulkQueryJpaFactory {
private BulkQueryJpaFactory() {}
static EntityManagerFactory createBulkQueryEntityManagerFactory(
ImmutableMap<String, String> cloudSqlConfigs) {
ParsedPersistenceXmlDescriptor descriptor =
PersistenceXmlUtility.getParsedPersistenceXmlDescriptor();
List<String> updatedManagedClasses =
Streams.concat(
descriptor.getManagedClassNames().stream(),
BulkQueryEntities.JPA_ENTITIES_NEW.stream())
.map(
name -> {
if (BulkQueryEntities.JPA_ENTITIES_REPLACEMENTS.containsKey(name)) {
return BulkQueryEntities.JPA_ENTITIES_REPLACEMENTS.get(name);
}
return name;
})
.collect(ImmutableList.toImmutableList());
descriptor.getManagedClassNames().clear();
descriptor.getManagedClassNames().addAll(updatedManagedClasses);
return Bootstrap.getEntityManagerFactoryBuilder(descriptor, cloudSqlConfigs).build();
}
public static JpaTransactionManager createBulkQueryJpaTransactionManager(
ImmutableMap<String, String> cloudSqlConfigs, Clock clock) {
return new JpaTransactionManagerImpl(
createBulkQueryEntityManagerFactory(cloudSqlConfigs), clock);
}
}

22
core/src/main/java/google/registry/persistence/PersistenceModule.java
View File

@@ -241,15 +241,6 @@ public abstract class PersistenceModule {
return new JpaTransactionManagerImpl(create(beamCloudSqlConfigs), clock);
}
@Provides
@Singleton
@BeamBulkQueryJpaTm
static JpaTransactionManager provideBeamBulkQueryJpaTm(
@BeamPipelineCloudSqlConfigs ImmutableMap<String, String> beamCloudSqlConfigs, Clock clock) {
return new JpaTransactionManagerImpl(
BulkQueryJpaFactory.createBulkQueryEntityManagerFactory(beamCloudSqlConfigs), clock);
}
@Provides
@Singleton
@NomulusToolJpaTm
@@ -371,11 +362,6 @@ public abstract class PersistenceModule {
public enum JpaTransactionManagerType {
/** The regular {@link JpaTransactionManager} for general use. */
REGULAR,
/**
* The {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities. Please
* see {@link google.registry.model.bulkquery.BulkQueryEntities} for more information.
*/
BULK_QUERY,
/**
* The {@link JpaTransactionManager} that uses the read-only Postgres replica if configured, or
* the standard DB if not.
@@ -398,14 +384,6 @@ public abstract class PersistenceModule {
@Documented
public @interface BeamJpaTm {}
/**
* Dagger qualifier for {@link JpaTransactionManager} that uses an alternative entity model for
* faster bulk queries.
*/
@Qualifier
@Documented
public @interface BeamBulkQueryJpaTm {}
/**
* Dagger qualifier for {@link JpaTransactionManager} used inside BEAM pipelines that uses the
* read-only Postgres replica if one is configured (otherwise it uses the standard DB).