1
0
mirror of https://github.com/google/nomulus synced 2026-05-21 07:11:48 +00:00

Compare commits

...

5 Commits

Author SHA1 Message Date
Weimin Yu
6249a8e118 Revise Host index on inet_addresses (#1549)
* Revise Host index on inet_addresses

The index on the 'inet_addresses array column should be of gin or gist
type, which index individual array elements. We use gin for now since
host updates are not often, and gin has better accuracy.

Since flyway script V108__... has not been deployed, we  edit the file
in place instead of adding a new script.

This will be followed up with a modified query that can take advantage
of the gin index. Until then we don't expect to see performance
improvement.

The suspected bottlenect query in the whois path is:

select * from "Host" where 'non-ip-string' = any(inet_address) and
deletion_time < now();

It needs to be revised into:

select * from "Host" where array['non-ip-string'] <@ inet_address and
deletion_time < now();

The combined change reduces the query time from 90ms to 30ms in Sandbox,
and from 150ms to 40ms in production.

It is unclear if this solves all problem with whois latency.
2022-03-08 14:22:01 -05:00
Ben McIlwain
9b7bb12cd1 Add deletionTime/inetAddresses indexes to Host table to support WHOIS (#1548)
* Add deletionTime/inetAddresses indexes to Host table to support WHOIS

Weimin identified these as missing, and being the cause of slowdowns in
NameserverLookupByIpCommand that we're seeing in sandbox.

This is the first of two PRs, adding just the Flyway/schema changes. The
second PR adding the Java object model changes is #1547.
2022-03-07 16:07:11 -05:00
Michael Muller
71d13bab71 Improve Transaction gap processing (#1546)
Skip multiple gaps in one pass and write the correct transaction id to
datastore.
2022-03-07 13:26:18 -05:00
Ben McIlwain
8db28b7e61 Add domainRepoId indexes to billing events (#1545)
* Add domainRepoId indexes to billing events #1544

The query analyzer identified this is a missing index on the BillingEvent table,
and I added it for recurrences and cancellations as well as it's likely to be a
problem for them too. "Give me all the billing events associated with a given
domain by its repo ID" seems like a pretty common use case for the DB (and does
appear to be used by our invoicing pipeline).

This is the first of two PRs that makes just the DB changes. The second PR
(#1544) will add the Java code changes, and will be committed after this one is
deployed.
2022-03-07 12:21:40 -05:00
Lai Jiang
5a7dc307c5 Update the the latest LTS Node version (#1543)
<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1543)
<!-- Reviewable:end -->
2022-03-07 11:49:18 -05:00
9 changed files with 770 additions and 637 deletions

View File

@@ -55,7 +55,7 @@ plugins {
node {
download = true
version = "14.15.5"
version = "16.14.0"
npmVersion = "6.14.11"
}

View File

@@ -127,12 +127,16 @@ public class ReplicateToDatastoreAction implements Runnable {
// Reload the last transaction id, which could possibly have changed.
LastSqlTransaction lastSqlTxn = LastSqlTransaction.load();
long nextTxnId = lastSqlTxn.getTransactionId() + 1;
if (nextTxnId < txnEntity.getId()) {
// Missing transaction id. This can happen normally. If a transaction gets
// rolled back, the sequence counter doesn't.
// Skip missing transactions. Missed transactions can happen normally. If a
// transaction gets rolled back, the sequence counter doesn't.
while (nextTxnId < txnEntity.getId()) {
logger.atWarning().log(
"Ignoring transaction %s, which does not exist.", nextTxnId);
} else if (nextTxnId > txnEntity.getId()) {
++nextTxnId;
}
if (nextTxnId > txnEntity.getId()) {
// We've already replayed this transaction. This shouldn't happen, as GAE cron
// is supposed to avoid overruns and this action shouldn't be executed from any
// other context, but it's not harmful as we can just ignore the transaction. Log

View File

@@ -208,18 +208,20 @@ public class ReplicateToDatastoreActionTest {
TestObject foo = TestObject.create("foo");
insertInDb(foo);
// Fail during the transaction to delete it.
try {
jpaTm()
.transact(
() -> {
jpaTm().delete(foo.key());
// Explicitly save the transaction entity to force the id update.
jpaTm().insert(new TransactionEntity(new byte[] {1, 2, 3}));
throw new RuntimeException("fail!!!");
});
} catch (Exception e) {
logger.atInfo().log("Got expected exception.");
// Fail two transactions.
for (int i = 0; i < 2; ++i) {
try {
jpaTm()
.transact(
() -> {
jpaTm().delete(foo.key());
// Explicitly save the transaction entity to force the id update.
jpaTm().insert(new TransactionEntity(new byte[] {1, 2, 3}));
throw new RuntimeException("fail!!!");
});
} catch (Exception e) {
logger.atInfo().log("Got expected exception.");
}
}
TestObject bar = TestObject.create("bar");
@@ -230,11 +232,13 @@ public class ReplicateToDatastoreActionTest {
assertThat(txns).hasSize(2);
for (TransactionEntity txn : txns) {
assertThat(txn.getId()).isNotEqualTo(2);
assertThat(txn.getId()).isNotEqualTo(3);
applyTransaction(txn);
}
assertThat(ofyTm().transact(() -> ofyTm().loadByKey(foo.key()))).isEqualTo(foo);
assertThat(ofyTm().transact(() -> ofyTm().loadByKey(bar.key()))).isEqualTo(bar);
assertThat(ofyTm().transact(() -> LastSqlTransaction.load()).getTransactionId()).isEqualTo(4);
}
@Test

File diff suppressed because it is too large Load Diff

View File

@@ -261,11 +261,11 @@ td.section {
</tr>
<tr>
<td class="property_name">generated on</td>
<td class="property_value">2022-03-04 22:54:56.709583</td>
<td class="property_value">2022-03-08 17:12:35.997985</td>
</tr>
<tr>
<td class="property_name">last flyway file</td>
<td id="lastFlywayFile" class="property_value">V106__add_missing_indexes_from_query_analyzer.sql</td>
<td id="lastFlywayFile" class="property_value">V108__add_host_indexes_for_whois.sql</td>
</tr>
</tbody>
</table>
@@ -284,7 +284,7 @@ td.section {
generated on
</text>
<text text-anchor="start" x="4755.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
2022-03-04 22:54:56.709583
2022-03-08 17:12:35.997985
</text>
<polygon fill="none" stroke="#888888" points="4668.02,-4 4668.02,-44 4933.02,-44 4933.02,-4 4668.02,-4" /> <!-- allocationtoken_a08ccbef -->
<g id="node1" class="node">
@@ -6923,6 +6923,18 @@ td.section {
<tr>
<td colspan="3"></td>
</tr>
<tr>
<td colspan="2" class="name">idxl8vobbecsd32k4ksavdfx8st6</td>
<td class="description right">[non-unique index]</td>
</tr>
<tr>
<td class="spacer"></td>
<td class="minwidth">domain_repo_id</td>
<td class="minwidth">ascending</td>
</tr>
<tr>
<td colspan="3"></td>
</tr>
<tr>
<td colspan="2" class="name">idxeokttmxtpq2hohcioe5t2242b</td>
<td class="description right">[non-unique index]</td>
@@ -7218,6 +7230,18 @@ td.section {
<tr>
<td colspan="3"></td>
</tr>
<tr>
<td colspan="2" class="name">idxbgfmveqa7e5hn689koikwn70r</td>
<td class="description right">[non-unique index]</td>
</tr>
<tr>
<td class="spacer"></td>
<td class="minwidth">domain_repo_id</td>
<td class="minwidth">ascending</td>
</tr>
<tr>
<td colspan="3"></td>
</tr>
<tr>
<td colspan="2" class="name">idx73l103vc5900ig3p4odf0cngt</td>
<td class="description right">[non-unique index]</td>
@@ -7454,6 +7478,18 @@ td.section {
<tr>
<td colspan="3"></td>
</tr>
<tr>
<td colspan="2" class="name">idxoqttafcywwdn41um6kwlt0n8b</td>
<td class="description right">[non-unique index]</td>
</tr>
<tr>
<td class="spacer"></td>
<td class="minwidth">domain_repo_id</td>
<td class="minwidth">ascending</td>
</tr>
<tr>
<td colspan="3"></td>
</tr>
<tr>
<td colspan="2" class="name">idxjny8wuot75b5e6p38r47wdawu</td>
<td class="description right">[non-unique index]</td>
@@ -11275,6 +11311,18 @@ td.section {
<tr>
<td colspan="3"></td>
</tr>
<tr>
<td colspan="2" class="name">idxrc77s1ndiemi2vwwudchye214</td>
<td class="description right">[non-unique index]</td>
</tr>
<tr>
<td class="spacer"></td>
<td class="minwidth">inet_addresses</td>
<td class="minwidth">unknown</td>
</tr>
<tr>
<td colspan="3"></td>
</tr>
<tr>
<td colspan="2" class="name">idxkpkh68n6dy5v51047yr6b0e9l</td>
<td class="description right">[non-unique index]</td>

View File

@@ -104,3 +104,5 @@ V103__creation_time_not_null.sql
V104__add_transfer_response_host_id_to_poll_message.sql
V105__add_index_on_host_name_in_host_table.sql
V106__add_missing_indexes_from_query_analyzer.sql
V107__add_billingevent_domainrepoid_indexes.sql
V108__add_host_indexes_for_whois.sql

View File

@@ -0,0 +1,18 @@
-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
CREATE INDEX IDXbgfmveqa7e5hn689koikwn70r ON "BillingEvent" (domain_repo_id);
CREATE INDEX IDXoqttafcywwdn41um6kwlt0n8b ON "BillingRecurrence" (domain_repo_id);
CREATE INDEX IDXl8vobbecsd32k4ksavdfx8st6 ON "BillingCancellation" (domain_repo_id);

View File

@@ -0,0 +1,16 @@
-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
CREATE INDEX IF NOT EXISTS IDXrc77s1ndiemi2vwwudchye214
ON "Host" USING GIN (inet_addresses);

View File

@@ -1665,6 +1665,13 @@ CREATE INDEX idxaro1omfuaxjwmotk3vo00trwm ON public."DomainHistory" USING btree
CREATE INDEX idxaydgox62uno9qx8cjlj5lauye ON public."PollMessage" USING btree (event_time);
--
-- Name: idxbgfmveqa7e5hn689koikwn70r; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX idxbgfmveqa7e5hn689koikwn70r ON public."BillingEvent" USING btree (domain_repo_id);
--
-- Name: idxbgssjudpm428mrv0xfpvgifps; Type: INDEX; Schema: public; Owner: -
--
@@ -1791,6 +1798,13 @@ CREATE INDEX idxkpkh68n6dy5v51047yr6b0e9l ON public."Host" USING btree (host_nam
CREATE INDEX idxku0fopwyvd57ebo8bf0jg9xo2 ON public."BillingCancellation" USING btree (billing_recurrence_id);
--
-- Name: idxl8vobbecsd32k4ksavdfx8st6; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX idxl8vobbecsd32k4ksavdfx8st6 ON public."BillingCancellation" USING btree (domain_repo_id);
--
-- Name: idxlrq7v63pc21uoh3auq6eybyhl; Type: INDEX; Schema: public; Owner: -
--
@@ -1833,6 +1847,13 @@ CREATE INDEX idxo1xdtpij2yryh0skxe9v91sep ON public."ContactHistory" USING btree
CREATE INDEX idxoqd7n4hbx86hvlgkilq75olas ON public."Contact" USING btree (contact_id);
--
-- Name: idxoqttafcywwdn41um6kwlt0n8b; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX idxoqttafcywwdn41um6kwlt0n8b ON public."BillingRecurrence" USING btree (domain_repo_id);
--
-- Name: idxp3usbtvk0v1m14i5tdp4xnxgc; Type: INDEX; Schema: public; Owner: -
--
@@ -1868,6 +1889,13 @@ CREATE INDEX idxqa3g92jc17e8dtiaviy4fet4x ON public."BillingCancellation" USING
CREATE INDEX idxr22ciyccwi9rrqmt1ro0s59qf ON public."Domain" USING btree (tech_contact);
--
-- Name: idxrc77s1ndiemi2vwwudchye214; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX idxrc77s1ndiemi2vwwudchye214 ON public."Host" USING gin (inet_addresses);
--
-- Name: idxrh4xmrot9bd63o382ow9ltfig; Type: INDEX; Schema: public; Owner: -
--