1
0
mirror of https://github.com/google/nomulus synced 2026-02-08 22:10:28 +00:00

Remove bulk query entities (#1834)

These alternative ORMs are introduced as a way to make querying large number of
domains and domain histories more efficient through bulk loading from several
to-be-joined tables separately, then in-memory re-assembly of the final entity,
bypassing the need to query multiple tables each time an entity is queried.

Their primary use case is loading these entities for comparison between
datastore and SQL during the migration, which has been completed. The
code remain unused as of now and their existence makes refactoring and
general maintenance more complicated than necessary due to the need to keep
them up to date.

Therefore we remove the related code.

<!-- Reviewable:start -->
- - -
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1834)
<!-- Reviewable:end -->
This commit is contained in:
Lai Jiang
2022-10-28 12:25:57 -04:00
committed by GitHub
parent 3f68ad5ea3
commit 1c90a6648e
15 changed files with 0 additions and 1088 deletions

View File

@@ -145,16 +145,4 @@ class RegistryPipelineOptionsTest {
.as(RegistryPipelineOptions.class);
assertThat(options.getJpaTransactionManagerType()).isEqualTo(JpaTransactionManagerType.REGULAR);
}
@Test
void jpaTransactionManagerType_bulkQueryJpa() {
RegistryPipelineOptions options =
PipelineOptionsFactory.fromArgs(
"--registryEnvironment=" + RegistryEnvironment.UNITTEST.name(),
"--jpaTransactionManagerType=BULK_QUERY")
.withValidation()
.as(RegistryPipelineOptions.class);
assertThat(options.getJpaTransactionManagerType())
.isEqualTo(JpaTransactionManagerType.BULK_QUERY);
}
}

View File

@@ -1,89 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static google.registry.model.bulkquery.BulkQueryEntities.assembleDomain;
import static google.registry.model.bulkquery.BulkQueryEntities.assembleDomainHistory;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import google.registry.model.domain.Domain;
import google.registry.model.domain.DomainHistory;
import google.registry.model.domain.DomainHistory.DomainHistoryId;
import google.registry.model.domain.GracePeriod;
import google.registry.model.domain.GracePeriod.GracePeriodHistory;
import google.registry.model.domain.secdns.DomainDsData;
import google.registry.model.domain.secdns.DomainDsDataHistory;
import google.registry.model.reporting.DomainTransactionRecord;
import google.registry.persistence.VKey;
/**
* Helpers for bulk-loading {@link Domain} and {@link google.registry.model.domain.DomainHistory}
* entities in <em>tests</em>.
*/
public class BulkQueryHelper {
static Domain loadAndAssembleDomain(String domainRepoId) {
return jpaTm()
.transact(
() ->
assembleDomain(
jpaTm().loadByKey(DomainLite.createVKey(domainRepoId)),
jpaTm()
.loadAllOfStream(GracePeriod.class)
.filter(gracePeriod -> gracePeriod.getDomainRepoId().equals(domainRepoId))
.collect(toImmutableSet()),
jpaTm()
.loadAllOfStream(DomainDsData.class)
.filter(dsData -> dsData.getDomainRepoId().equals(domainRepoId))
.collect(toImmutableSet()),
jpaTm()
.loadAllOfStream(DomainHost.class)
.filter(domainHost -> domainHost.getDomainRepoId().equals(domainRepoId))
.map(DomainHost::getHostVKey)
.collect(toImmutableSet())));
}
static DomainHistory loadAndAssembleDomainHistory(DomainHistoryId domainHistoryId) {
return jpaTm()
.transact(
() ->
assembleDomainHistory(
jpaTm().loadByKey(VKey.createSql(DomainHistoryLite.class, domainHistoryId)),
jpaTm()
.loadAllOfStream(DomainDsDataHistory.class)
.filter(
domainDsDataHistory ->
domainDsDataHistory.getDomainHistoryId().equals(domainHistoryId))
.collect(toImmutableSet()),
jpaTm()
.loadAllOfStream(DomainHistoryHost.class)
.filter(
domainHistoryHost ->
domainHistoryHost.getDomainHistoryId().equals(domainHistoryId))
.map(DomainHistoryHost::getHostVKey)
.collect(toImmutableSet()),
jpaTm()
.loadAllOfStream(GracePeriodHistory.class)
.filter(
gracePeriodHistory ->
gracePeriodHistory.getDomainHistoryId().equals(domainHistoryId))
.collect(toImmutableSet()),
jpaTm()
.loadAllOfStream(DomainTransactionRecord.class)
.filter(x -> true)
.collect(toImmutableSet())));
}
}

View File

@@ -1,125 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static org.joda.time.DateTimeZone.UTC;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
import com.google.common.truth.Truth8;
import google.registry.model.domain.DomainHistory;
import google.registry.testing.AppEngineExtension;
import google.registry.testing.FakeClock;
import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.metamodel.Attribute;
import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for {@link DomainHistoryLite}. */
public class DomainHistoryLiteTest {
protected FakeClock fakeClock = new FakeClock(DateTime.now(UTC));
@RegisterExtension
public final AppEngineExtension appEngine =
AppEngineExtension.builder().withCloudSql().withClock(fakeClock).build();
private final TestSetupHelper setupHelper = new TestSetupHelper(fakeClock);
@BeforeEach
void setUp() {
setupHelper.initializeAllEntities();
}
@AfterEach
void afterEach() {
setupHelper.tearDownBulkQueryJpaTm();
}
@Test
void readDomainHistoryHost() {
setupHelper.applyChangeToDomainAndHistory();
setupHelper.setupBulkQueryJpaTm(appEngine);
Truth8.assertThat(
jpaTm().transact(() -> jpaTm().loadAllOf(DomainHistoryHost.class)).stream()
.map(DomainHistoryHost::getHostVKey))
.containsExactly(setupHelper.host.createVKey());
}
@Test
void domainHistoryLiteAttributes_versusDomainHistory() {
Set<String> domainHistoryAttributes =
jpaTm()
.transact(
() ->
jpaTm()
.getEntityManager()
.getMetamodel()
.entity(DomainHistory.class)
.getAttributes())
.stream()
.map(Attribute::getName)
.collect(Collectors.toSet());
setupHelper.setupBulkQueryJpaTm(appEngine);
Set<String> domainHistoryLiteAttributes =
jpaTm()
.transact(
() ->
jpaTm()
.getEntityManager()
.getMetamodel()
.entity(DomainHistoryLite.class)
.getAttributes())
.stream()
.map(Attribute::getName)
.collect(Collectors.toSet());
assertThat(domainHistoryAttributes).containsAtLeastElementsIn(domainHistoryLiteAttributes);
SetView<?> excludedFromDomainHistory =
Sets.difference(domainHistoryAttributes, domainHistoryLiteAttributes);
assertThat(excludedFromDomainHistory)
.containsExactly(
"dsDataHistories",
"gracePeriodHistories",
"internalDomainTransactionRecords",
"nsHosts");
}
@Test
void readDomainHistory_noContent() {
setupHelper.setupBulkQueryJpaTm(appEngine);
assertThat(
BulkQueryHelper.loadAndAssembleDomainHistory(
setupHelper.domainHistory.getDomainHistoryId()))
.isEqualTo(setupHelper.domainHistory);
}
@Test
void readDomainHistory_full() {
setupHelper.applyChangeToDomainAndHistory();
setupHelper.setupBulkQueryJpaTm(appEngine);
assertThat(
BulkQueryHelper.loadAndAssembleDomainHistory(
setupHelper.domainHistory.getDomainHistoryId()))
.isEqualTo(setupHelper.domainHistory);
}
}

View File

@@ -1,111 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.bulkquery.BulkQueryHelper.loadAndAssembleDomain;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static org.joda.time.DateTimeZone.UTC;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
import com.google.common.truth.Truth8;
import google.registry.model.domain.Domain;
import google.registry.testing.AppEngineExtension;
import google.registry.testing.FakeClock;
import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.metamodel.Attribute;
import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for reading {@link DomainLite}. */
class DomainLiteTest {
protected FakeClock fakeClock = new FakeClock(DateTime.now(UTC));
@RegisterExtension
public final AppEngineExtension appEngine =
AppEngineExtension.builder().withCloudSql().withClock(fakeClock).build();
private final TestSetupHelper setupHelper = new TestSetupHelper(fakeClock);
@BeforeEach
void setUp() {
setupHelper.initializeAllEntities();
}
@AfterEach
void afterEach() {
setupHelper.tearDownBulkQueryJpaTm();
}
@Test
void readDomainHost() {
setupHelper.applyChangeToDomainAndHistory();
setupHelper.setupBulkQueryJpaTm(appEngine);
Truth8.assertThat(
jpaTm().transact(() -> jpaTm().loadAllOf(DomainHost.class)).stream()
.map(DomainHost::getHostVKey))
.containsExactly(setupHelper.host.createVKey());
}
@Test
void domainLiteAttributes_versusDomain() {
Set<String> domainAttributes =
jpaTm()
.transact(
() ->
jpaTm().getEntityManager().getMetamodel().entity(Domain.class).getAttributes())
.stream()
.map(Attribute::getName)
.collect(Collectors.toSet());
setupHelper.setupBulkQueryJpaTm(appEngine);
Set<String> domainLiteAttributes =
jpaTm()
.transact(
() ->
jpaTm()
.getEntityManager()
.getMetamodel()
.entity(DomainLite.class)
.getAttributes())
.stream()
.map(Attribute::getName)
.collect(Collectors.toSet());
assertThat(domainAttributes).containsAtLeastElementsIn(domainLiteAttributes);
SetView<?> excludedFromDomain = Sets.difference(domainAttributes, domainLiteAttributes);
assertThat(excludedFromDomain)
.containsExactly("internalDelegationSignerData", "internalGracePeriods", "nsHosts");
}
@Test
void readDomainLite_simple() {
setupHelper.setupBulkQueryJpaTm(appEngine);
assertThat(loadAndAssembleDomain(TestSetupHelper.DOMAIN_REPO_ID)).isEqualTo(setupHelper.domain);
}
@Test
void readDomainLite_full() {
setupHelper.applyChangeToDomainAndHistory();
setupHelper.setupBulkQueryJpaTm(appEngine);
assertThat(loadAndAssembleDomain(TestSetupHelper.DOMAIN_REPO_ID)).isEqualTo(setupHelper.domain);
}
}

View File

@@ -1,219 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.bulkquery;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.testing.SqlHelper.saveRegistrar;
import static google.registry.util.DateTimeUtils.END_OF_TIME;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableSet;
import google.registry.model.contact.Contact;
import google.registry.model.domain.DesignatedContact;
import google.registry.model.domain.Domain;
import google.registry.model.domain.DomainAuthInfo;
import google.registry.model.domain.DomainHistory;
import google.registry.model.domain.GracePeriod;
import google.registry.model.domain.Period;
import google.registry.model.domain.launch.LaunchNotice;
import google.registry.model.domain.rgp.GracePeriodStatus;
import google.registry.model.domain.secdns.DomainDsData;
import google.registry.model.eppcommon.AuthInfo.PasswordAuth;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.eppcommon.Trid;
import google.registry.model.host.Host;
import google.registry.model.registrar.Registrar;
import google.registry.model.reporting.DomainTransactionRecord;
import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.tld.Registry;
import google.registry.model.transfer.ContactTransferData;
import google.registry.persistence.BulkQueryJpaFactory;
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.persistence.transaction.TransactionManagerFactory;
import google.registry.testing.AppEngineExtension;
import google.registry.testing.DatabaseHelper;
import google.registry.testing.FakeClock;
/**
* Utilities for managing domain-related SQL test scenarios.
*
* <p>The {@link #initializeAllEntities} method initializes the database, and the {@link
* #applyChangeToDomainAndHistory} makes one change to the domain, generating an additional history
* event. The most up-to-date values of the relevant entities are saved in public instance
* variables, {@link #domain} etc.
*
* <p>This class makes use of {@link DatabaseHelper}, which requires Datastore. Tests that use this
* class should use {@link AppEngineExtension}.
*/
public final class TestSetupHelper {
public static final String TLD = "tld";
public static final String DOMAIN_REPO_ID = "4-TLD";
public static final String DOMAIN_NAME = "example.tld";
public static final String REGISTRAR_ID = "AnRegistrar";
private final FakeClock fakeClock;
public Registry registry;
public Registrar registrar;
public Contact contact;
public Domain domain;
public DomainHistory domainHistory;
public Host host;
private JpaTransactionManager originalJpaTm;
private JpaTransactionManager bulkQueryJpaTm;
public TestSetupHelper(FakeClock fakeClock) {
this.fakeClock = fakeClock;
}
public void initializeAllEntities() {
registry = putInDb(DatabaseHelper.newRegistry(TLD, Ascii.toUpperCase(TLD)));
registrar = saveRegistrar(REGISTRAR_ID);
contact = putInDb(createContact(DOMAIN_REPO_ID, REGISTRAR_ID));
domain = putInDb(createSimpleDomain(contact));
domainHistory = putInDb(createHistoryWithoutContent(domain, fakeClock));
host = putInDb(createHost());
}
public void applyChangeToDomainAndHistory() {
domain = putInDb(createFullDomain(contact, host, fakeClock));
domainHistory = putInDb(createFullHistory(domain, fakeClock));
}
public void setupBulkQueryJpaTm(AppEngineExtension appEngineExtension) {
bulkQueryJpaTm =
BulkQueryJpaFactory.createBulkQueryJpaTransactionManager(
appEngineExtension
.getJpaIntegrationTestExtension()
.map(JpaIntegrationTestExtension::getJpaProperties)
.orElseThrow(
() -> new IllegalStateException("Expecting JpaIntegrationTestExtension.")),
fakeClock);
originalJpaTm = TransactionManagerFactory.jpaTm();
TransactionManagerFactory.setJpaTm(() -> bulkQueryJpaTm);
}
public void tearDownBulkQueryJpaTm() {
if (bulkQueryJpaTm != null) {
bulkQueryJpaTm.teardown();
TransactionManagerFactory.setJpaTm(() -> originalJpaTm);
}
}
static Contact createContact(String repoId, String registrarId) {
return new Contact.Builder()
.setRepoId(repoId)
.setCreationRegistrarId(registrarId)
.setTransferData(new ContactTransferData.Builder().build())
.setPersistedCurrentSponsorRegistrarId(registrarId)
.build();
}
static Domain createSimpleDomain(Contact contact) {
return DatabaseHelper.newDomain(DOMAIN_NAME, DOMAIN_REPO_ID, contact)
.asBuilder()
.setCreationRegistrarId(REGISTRAR_ID)
.setPersistedCurrentSponsorRegistrarId(REGISTRAR_ID)
.build();
}
static Domain createFullDomain(Contact contact, Host host, FakeClock fakeClock) {
return createSimpleDomain(contact)
.asBuilder()
.setDomainName(DOMAIN_NAME)
.setRepoId(DOMAIN_REPO_ID)
.setCreationRegistrarId(REGISTRAR_ID)
.setLastEppUpdateTime(fakeClock.nowUtc())
.setLastEppUpdateRegistrarId(REGISTRAR_ID)
.setLastTransferTime(fakeClock.nowUtc())
.setNameservers(host.createVKey())
.setStatusValues(
ImmutableSet.of(
StatusValue.CLIENT_DELETE_PROHIBITED,
StatusValue.SERVER_DELETE_PROHIBITED,
StatusValue.SERVER_TRANSFER_PROHIBITED,
StatusValue.SERVER_UPDATE_PROHIBITED,
StatusValue.SERVER_RENEW_PROHIBITED,
StatusValue.SERVER_HOLD))
.setContacts(
ImmutableSet.of(
DesignatedContact.create(DesignatedContact.Type.ADMIN, contact.createVKey())))
.setSubordinateHosts(ImmutableSet.of("ns1.example.com"))
.setPersistedCurrentSponsorRegistrarId(REGISTRAR_ID)
.setRegistrationExpirationTime(fakeClock.nowUtc().plusYears(1))
.setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("password")))
.setDsData(ImmutableSet.of(DomainDsData.create(1, 2, 3, new byte[] {0, 1, 2})))
.setLaunchNotice(LaunchNotice.create("tcnid", "validatorId", START_OF_TIME, START_OF_TIME))
.setSmdId("smdid")
.addGracePeriod(
GracePeriod.create(
GracePeriodStatus.ADD, DOMAIN_REPO_ID, END_OF_TIME, REGISTRAR_ID, null, 100L))
.build();
}
static Host createHost() {
return new Host.Builder()
.setRepoId("host1")
.setHostName("ns1.example.com")
.setCreationRegistrarId(REGISTRAR_ID)
.setPersistedCurrentSponsorRegistrarId(REGISTRAR_ID)
.build();
}
static DomainTransactionRecord createDomainTransactionRecord(FakeClock fakeClock) {
return new DomainTransactionRecord.Builder()
.setTld(TLD)
.setReportingTime(fakeClock.nowUtc())
.setReportField(TransactionReportField.NET_ADDS_1_YR)
.setReportAmount(1)
.build();
}
static DomainHistory createHistoryWithoutContent(Domain domain, FakeClock fakeClock) {
return new DomainHistory.Builder()
.setType(HistoryEntry.Type.DOMAIN_CREATE)
.setXmlBytes("<xml></xml>".getBytes(UTF_8))
.setModificationTime(fakeClock.nowUtc())
.setRegistrarId(REGISTRAR_ID)
.setTrid(Trid.create("ABC-123", "server-trid"))
.setBySuperuser(false)
.setReason("reason")
.setRequestedByRegistrar(true)
.setDomainRepoId(domain.getRepoId())
.setOtherRegistrarId("otherClient")
.setPeriod(Period.create(1, Period.Unit.YEARS))
.build();
}
static DomainHistory createFullHistory(Domain domain, FakeClock fakeClock) {
return createHistoryWithoutContent(domain, fakeClock)
.asBuilder()
.setType(HistoryEntry.Type.DOMAIN_TRANSFER_APPROVE)
.setDomain(domain)
.setDomainTransactionRecords(ImmutableSet.of(createDomainTransactionRecord(fakeClock)))
.build();
}
static <T> T putInDb(T entity) {
jpaTm().transact(() -> jpaTm().put(entity));
return jpaTm().transact(() -> jpaTm().loadByEntity(entity));
}
}