From e7916080986aecc960caaf760d02af3984894840 Mon Sep 17 00:00:00 2001 From: sarahcaseybot Date: Wed, 1 May 2024 11:44:08 -0400 Subject: [PATCH] Add more indexes to speed up deleteProberDataAction (#2423) This adds an index on transfer_billing_cancellation_id to Domain and superordinate_domain to Host. When tested on crash with the action limited to only delete 10,000 domains, before these indexes were added the action took about 2 hours to delete 10,000 domains. Once these indexes were added, the action was able to delete the 10,000 domains in a little under 2 minutes. --- .../google/registry/model/domain/Domain.java | 3 +- .../java/google/registry/model/host/Host.java | 1 + .../sql/er_diagram/brief_er_diagram.html | 6 ++-- .../sql/er_diagram/full_er_diagram.html | 31 +++++++++++++++++-- db/src/main/resources/sql/flyway.txt | 1 + ..._indexes_needed_for_delete_prober_data.sql | 16 ++++++++++ .../sql/schema/db-schema.sql.generated | 2 ++ .../resources/sql/schema/nomulus.golden.sql | 14 +++++++++ 8 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 db/src/main/resources/sql/flyway/V169__add_more_indexes_needed_for_delete_prober_data.sql diff --git a/core/src/main/java/google/registry/model/domain/Domain.java b/core/src/main/java/google/registry/model/domain/Domain.java index 04194dfba..d92e030c7 100644 --- a/core/src/main/java/google/registry/model/domain/Domain.java +++ b/core/src/main/java/google/registry/model/domain/Domain.java @@ -61,7 +61,8 @@ import org.joda.time.DateTime; @Index(columnList = "lordnPhase"), @Index(columnList = "billing_recurrence_id"), @Index(columnList = "transfer_billing_event_id"), - @Index(columnList = "transfer_billing_recurrence_id") + @Index(columnList = "transfer_billing_recurrence_id"), + @Index(columnList = "transfer_billing_cancellation_id") }) @WithVKey(String.class) @ExternalMessagingName("domain") diff --git a/core/src/main/java/google/registry/model/host/Host.java b/core/src/main/java/google/registry/model/host/Host.java index cb6c3b556..0ee095cf2 100644 --- a/core/src/main/java/google/registry/model/host/Host.java +++ b/core/src/main/java/google/registry/model/host/Host.java @@ -49,6 +49,7 @@ import javax.persistence.Table; @Index(columnList = "creationTime"), @Index(columnList = "deletionTime"), @Index(columnList = "currentSponsorRegistrarId"), + @Index(columnList = "superordinateDomain") }) @ExternalMessagingName("host") @WithVKey(String.class) diff --git a/db/src/main/resources/sql/er_diagram/brief_er_diagram.html b/db/src/main/resources/sql/er_diagram/brief_er_diagram.html index 0f9a2508a..921a68a3f 100644 --- a/db/src/main/resources/sql/er_diagram/brief_er_diagram.html +++ b/db/src/main/resources/sql/er_diagram/brief_er_diagram.html @@ -261,11 +261,11 @@ td.section { generated on - 2024-04-29 19:20:24.144414867 + 2024-04-30 21:17:11.765649774 last flyway file - V168__registry_lock_email.sql + V169__add_more_indexes_needed_for_delete_prober_data.sql @@ -280,7 +280,7 @@ td.section { generated by SchemaCrawler 16.10.1 generated on - 2024-04-29 19:20:24.144414867 + 2024-04-30 21:17:11.765649774 diff --git a/db/src/main/resources/sql/er_diagram/full_er_diagram.html b/db/src/main/resources/sql/er_diagram/full_er_diagram.html index b6511ae9c..9418418c3 100644 --- a/db/src/main/resources/sql/er_diagram/full_er_diagram.html +++ b/db/src/main/resources/sql/er_diagram/full_er_diagram.html @@ -261,11 +261,11 @@ td.section { </tr> <tr> <td class="property_name">generated on</td> - <td class="property_value">2024-04-29 19:20:21.522169488</td> + <td class="property_value">2024-04-30 21:17:04.536352392</td> </tr> <tr> <td class="property_name">last flyway file</td> - <td id="lastFlywayFile" class="property_value">V168__registry_lock_email.sql</td> + <td id="lastFlywayFile" class="property_value">V169__add_more_indexes_needed_for_delete_prober_data.sql</td> </tr> </tbody> </table> @@ -280,7 +280,8 @@ td.section { <text text-anchor="start" x="4443.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated by</text> <text text-anchor="start" x="4526.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">SchemaCrawler 16.10.1</text> <text text-anchor="start" x="4442.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated on</text> - <text text-anchor="start" x="4526.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">2024-04-29 19:20:21.522169488</text> + + <text text-anchor="start" x="4526.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">2024-04-30 21:17:04.536352392</text> <polygon fill="none" stroke="#888888" points="4439,-4 4439,-44 4726,-44 4726,-4 4439,-4" /> <!-- allocationtoken_a08ccbef --> <g id="node1" class="node"> <title> @@ -7700,6 +7701,18 @@ td.section { <tr> <td colspan="3"></td> </tr> + <tr> + <td colspan="2" class="name">idxhteajcrxmq4o8rsys8kevyiqr</td> + <td class="description right">[non-unique index]</td> + </tr> + <tr> + <td class="spacer"></td> + <td class="minwidth">transfer_billing_cancellation_id</td> + <td class="minwidth">ascending</td> + </tr> + <tr> + <td colspan="3"></td> + </tr> <tr> <td colspan="2" class="name">idx3y3k7m2bkgahm9sixiohgyrga</td> <td class="description right">[non-unique index]</td> @@ -9467,6 +9480,18 @@ td.section { <tr> <td colspan="3"></td> </tr> + <tr> + <td colspan="2" class="name">idxorp4yv9ult4ds6kgxo5fs5gnw</td> + <td class="description right">[non-unique index]</td> + </tr> + <tr> + <td class="spacer"></td> + <td class="minwidth">superordinate_domain</td> + <td class="minwidth">ascending</td> + </tr> + <tr> + <td colspan="3"></td> + </tr> <tr> <td colspan="2" class="name">idxy98mebut8ix1v07fjxxdkqcx</td> <td class="description right">[non-unique index]</td> diff --git a/db/src/main/resources/sql/flyway.txt b/db/src/main/resources/sql/flyway.txt index 5f53b6842..84addfdbd 100644 --- a/db/src/main/resources/sql/flyway.txt +++ b/db/src/main/resources/sql/flyway.txt @@ -166,3 +166,4 @@ V165__add_domain_repo_id_indexes_to_more_tables.sql V166__drop_should_publish_column.sql V167__creat_billing_cost_transitions_not_null.sql V168__registry_lock_email.sql +V169__add_more_indexes_needed_for_delete_prober_data.sql diff --git a/db/src/main/resources/sql/flyway/V169__add_more_indexes_needed_for_delete_prober_data.sql b/db/src/main/resources/sql/flyway/V169__add_more_indexes_needed_for_delete_prober_data.sql new file mode 100644 index 000000000..38de04d9f --- /dev/null +++ b/db/src/main/resources/sql/flyway/V169__add_more_indexes_needed_for_delete_prober_data.sql @@ -0,0 +1,16 @@ +-- Copyright 2024 The Nomulus Authors. All Rights Reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE INDEX CONCURRENTLY IF NOT EXISTS IDXhteajcrxmq4o8rsys8kevyiqr ON "Domain" (transfer_billing_cancellation_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS IDXorp4yv9ult4ds6kgxo5fs5gnw ON "Host" (superordinate_domain); diff --git a/db/src/main/resources/sql/schema/db-schema.sql.generated b/db/src/main/resources/sql/schema/db-schema.sql.generated index 27332b18a..5e0a3dbfe 100644 --- a/db/src/main/resources/sql/schema/db-schema.sql.generated +++ b/db/src/main/resources/sql/schema/db-schema.sql.generated @@ -962,6 +962,7 @@ create index IDXnjhib7v6fj7dhj5qydkefkl2u on "Domain" (lordn_phase); create index IDXsfci08jgsymxy6ovh4k7r358c on "Domain" (billing_recurrence_id); create index IDX3y3k7m2bkgahm9sixiohgyrga on "Domain" (transfer_billing_event_id); create index IDXcju58vqascbpve1t7fem53ctl on "Domain" (transfer_billing_recurrence_id); +create index IDXhteajcrxmq4o8rsys8kevyiqr on "Domain" (transfer_billing_cancellation_id); create index IDXrh4xmrot9bd63o382ow9ltfig on "DomainHistory" (creation_time); create index IDXaro1omfuaxjwmotk3vo00trwm on "DomainHistory" (history_registrar_id); create index IDXsu1nam10cjes9keobapn5jvxj on "DomainHistory" (history_type); @@ -982,6 +983,7 @@ create index IDXkpkh68n6dy5v51047yr6b0e9l on "Host" (host_name); create index IDXy98mebut8ix1v07fjxxdkqcx on "Host" (creation_time); create index IDXovmntef6l45tw2bsfl56tcugx on "Host" (deletion_time); create index IDXl49vydnq0h5j1piefwjy4i8er on "Host" (current_sponsor_registrar_id); +create index IDXorp4yv9ult4ds6kgxo5fs5gnw on "Host" (superordinate_domain); create index IDXfg2nnjlujxo6cb9fha971bq2n on "HostHistory" (creation_time); create index IDX1iy7njgb7wjmj9piml4l2g0qi on "HostHistory" (history_registrar_id); create index IDXkkwbwcwvrdkkqothkiye4jiff on "HostHistory" (host_name); diff --git a/db/src/main/resources/sql/schema/nomulus.golden.sql b/db/src/main/resources/sql/schema/nomulus.golden.sql index 68e5aaf3d..b2132bd05 100644 --- a/db/src/main/resources/sql/schema/nomulus.golden.sql +++ b/db/src/main/resources/sql/schema/nomulus.golden.sql @@ -2185,6 +2185,13 @@ CREATE INDEX idxhmv411mdqo5ibn4vy7ykxpmlv ON public."BillingEvent" USING btree ( CREATE INDEX idxhp33wybmb6tbpr1bq7ttwk8je ON public."ContactHistory" USING btree (history_registrar_id); +-- +-- Name: idxhteajcrxmq4o8rsys8kevyiqr; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idxhteajcrxmq4o8rsys8kevyiqr ON public."Domain" USING btree (transfer_billing_cancellation_id); + + -- -- Name: idxiahqo1d1fqdfknywmj2xbxl7t; Type: INDEX; Schema: public; Owner: - -- @@ -2339,6 +2346,13 @@ CREATE INDEX idxoqd7n4hbx86hvlgkilq75olas ON public."Contact" USING btree (conta CREATE INDEX idxoqttafcywwdn41um6kwlt0n8b ON public."BillingRecurrence" USING btree (domain_repo_id); +-- +-- Name: idxorp4yv9ult4ds6kgxo5fs5gnw; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idxorp4yv9ult4ds6kgxo5fs5gnw ON public."Host" USING btree (superordinate_domain); + + -- -- Name: idxovmntef6l45tw2bsfl56tcugx; Type: INDEX; Schema: public; Owner: - --