From 3006ca39ca925e4a379100d7333879da4243b0f0 Mon Sep 17 00:00:00 2001 From: Michael Muller Date: Fri, 25 Mar 2022 12:41:10 -0400 Subject: [PATCH] Add a "list_txns" to dump Transaction table (#1569) * Add a "list_txns" to dump Transaction table Add the list_txns command which can dump the entire contents of the Transaction table, either in csv format or as human readable transactions. The CSV format is useful for storing the transaction table at a specific point in time for later reference without requiring us to repeatedly hit the replica. Creating this without tests because this command has a very short shelf-life and is really only intended to be run by developers. Tested all features locally. * Reformatted --- .../registry/tools/ListTxnsCommand.java | 116 ++++++++++++++++++ .../google/registry/tools/RegistryCli.java | 2 + .../google/registry/tools/RegistryTool.java | 1 + .../registry/tools/RegistryToolComponent.java | 4 + 4 files changed, 123 insertions(+) create mode 100644 core/src/main/java/google/registry/tools/ListTxnsCommand.java diff --git a/core/src/main/java/google/registry/tools/ListTxnsCommand.java b/core/src/main/java/google/registry/tools/ListTxnsCommand.java new file mode 100644 index 000000000..b50f66a7f --- /dev/null +++ b/core/src/main/java/google/registry/tools/ListTxnsCommand.java @@ -0,0 +1,116 @@ +// Copyright 2022 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.base.Splitter; +import com.google.common.io.BaseEncoding; +import google.registry.model.common.Cursor; +import google.registry.persistence.transaction.Transaction; +import google.registry.persistence.transaction.TransactionEntity; +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + +/** Lists {@link Cursor} timestamps used by locking rolling cursor tasks, like in RDE. */ +@Parameters(separators = " =", commandDescription = "Lists the contents of the Transaction table.") +final class ListTxnsCommand implements CommandWithRemoteApi { + + @Parameter(names = "--infile", description = "Parse an input file instead of reading from db.") + private String infile; + + @Parameter( + names = "--full_dump", + description = + "Do a full dump of the contents of the transaction. Without this, " + + "just write transactions as CSV lines suitable for ingestion via --infile.") + private boolean fullDump = false; + + @Override + public void run() { + if (infile == null) { + fetchFromDb(); + } else { + parseCsvFile(); + } + } + + private void parseCsvFile() { + try { + BufferedReader src = Files.newBufferedReader(Paths.get(infile), UTF_8); + String line; + while ((line = src.readLine()) != null) { + List cols = Splitter.on(",").splitToList(line); + writeRecord(Integer.parseInt(cols.get(0)), BaseEncoding.base64().decode(cols.get(1))); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void fetchFromDb() { + long lastTransactionId = 0; + List results; + + do { + final long txnId = lastTransactionId; // For use in the lambda. + results = + replicaJpaTm() + .transact( + () -> + replicaJpaTm() + .query( + "select t from TransactionEntity t where id > :lastTransactionId", + TransactionEntity.class) + .setParameter("lastTransactionId", txnId) + .setMaxResults(1000) + .getResultList()); + + for (TransactionEntity txn : results) { + writeRecord(txn.getId(), txn.getContents()); + lastTransactionId = txn.getId(); + } + } while (results.size() > 0); + } + + private void writeRecord(long id, byte[] contents) { + if (fullDump) { + Transaction txn; + try { + txn = Transaction.deserialize(contents); + } catch (IOException ex) { + System.err.printf("Error deserializing transaction %s\n", id); + return; + } + System.out.printf("transaction %s <<<\n", id); + for (Transaction.Mutation mut : txn.getMutations()) { + if (mut instanceof Transaction.Update) { + System.out.println("updating: " + ((Transaction.Update) mut).getEntity()); + } else { + System.out.println("deleting: " + ((Transaction.Delete) mut).getKey()); + } + } + System.out.println(">>>"); + } else { + System.out.printf("%s,%s\n", id, BaseEncoding.base64().encode(contents)); + } + } +} diff --git a/core/src/main/java/google/registry/tools/RegistryCli.java b/core/src/main/java/google/registry/tools/RegistryCli.java index 20e41dbf5..4eacacb7e 100644 --- a/core/src/main/java/google/registry/tools/RegistryCli.java +++ b/core/src/main/java/google/registry/tools/RegistryCli.java @@ -250,6 +250,8 @@ final class RegistryCli implements AutoCloseable, CommandRunner { // Cloud SQL after the database migration. Note that the DB password is stored in Datastore // and it is already initialized above. TransactionManagerFactory.setJpaTm(() -> component.nomulusToolJpaTransactionManager().get()); + TransactionManagerFactory.setReplicaJpaTm( + () -> component.nomulusToolReplicaJpaTransactionManager().get()); } command.run(); diff --git a/core/src/main/java/google/registry/tools/RegistryTool.java b/core/src/main/java/google/registry/tools/RegistryTool.java index 2447c2e5b..54e64e459 100644 --- a/core/src/main/java/google/registry/tools/RegistryTool.java +++ b/core/src/main/java/google/registry/tools/RegistryTool.java @@ -92,6 +92,7 @@ public final class RegistryTool { .put("list_registrars", ListRegistrarsCommand.class) .put("list_reserved_lists", ListReservedListsCommand.class) .put("list_tlds", ListTldsCommand.class) + .put("list_txns", ListTxnsCommand.class) .put("load_snapshot", LoadSnapshotCommand.class) .put("load_test", LoadTestCommand.class) .put("lock_domain", LockDomainCommand.class) diff --git a/core/src/main/java/google/registry/tools/RegistryToolComponent.java b/core/src/main/java/google/registry/tools/RegistryToolComponent.java index 6429acd73..042a97578 100644 --- a/core/src/main/java/google/registry/tools/RegistryToolComponent.java +++ b/core/src/main/java/google/registry/tools/RegistryToolComponent.java @@ -33,6 +33,7 @@ import google.registry.keyring.api.KeyModule; import google.registry.keyring.kms.KmsModule; import google.registry.persistence.PersistenceModule; import google.registry.persistence.PersistenceModule.NomulusToolJpaTm; +import google.registry.persistence.PersistenceModule.ReadOnlyReplicaJpaTm; import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.privileges.secretmanager.SecretManagerModule; import google.registry.rde.RdeModule; @@ -187,6 +188,9 @@ interface RegistryToolComponent { @NomulusToolJpaTm Lazy nomulusToolJpaTransactionManager(); + @ReadOnlyReplicaJpaTm + Lazy nomulusToolReplicaJpaTransactionManager(); + @Component.Builder interface Builder { @BindsInstance