mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-09-12 02:56:26 +00:00
Code Issues Releases Wiki Activity

Upgrade CompareDbBackup for Datastore V3 (#543)

Browse Source 
* Upgrade CompareDbBackup for Datastore V3

Upgrade the CompareDbBackup class to work with latest
Datastore backup directory structure.

Also fixed a few unrelated minor issues:
- Remaining cases of improper use of System.setOut
- Wrong import order in one class
This commit is contained in:
Weimin Yu
2020-04-06 10:50:38 -04:00
committed by GitHub
parent 1ded33ecea
commit 3a7d71e411
9 changed files with 57 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/google/registry/tools/CompareDbBackups.java
+22 -3
View File
@@ -18,9 +18,20 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
import java.io.File;
import java.util.function.Predicate;
/** Compare two database backups. */
/**
* Compares two Datastore backups in V3 format on local file system. This is for use in tests and
* experiments with small data sizes.
*
* <p>This utility only supports the current Datastore backup format (version 3). A backup is a
* two-level directory hierarchy with data files in level-db format (output-*) and Datastore
* metadata files (*.export_metadata).
*/
class CompareDbBackups {
private static final String DS_V3_BACKUP_FILE_PREFIX = "output-";
private static final Predicate<File> DATA_FILE_MATCHER =
file -> file.isFile() && file.getName().startsWith(DS_V3_BACKUP_FILE_PREFIX);
public static void main(String[] args) {
if (args.length != 2) {
@@ -29,9 +40,13 @@ class CompareDbBackups {
}
ImmutableSet<ComparableEntity> entities1 =
new RecordAccumulator().readDirectory(new File(args[0])).getComparableEntitySet();
new RecordAccumulator()
.readDirectory(new File(args[0]), DATA_FILE_MATCHER)
.getComparableEntitySet();
ImmutableSet<ComparableEntity> entities2 =
new RecordAccumulator().readDirectory(new File(args[1])).getComparableEntitySet();
new RecordAccumulator()
.readDirectory(new File(args[1]), DATA_FILE_MATCHER)
.getComparableEntitySet();
// Calculate the entities added and removed.
SetView<ComparableEntity> added = Sets.difference(entities2, entities1);
@@ -54,6 +69,10 @@ class CompareDbBackups {
System.out.println(entity);
}
}
if (added.isEmpty() && removed.isEmpty()) {
System.out.printf("\nBoth sets have the same %d entities.\n", entities1.size());
}
}
/** Print out multi-line text in a pretty ASCII header frame. */
core/src/main/java/google/registry/tools/RecordAccumulator.java
+5 -4
View File
@@ -20,17 +20,18 @@ import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.function.Predicate;
/** Utility class that accumulates Entity records from level db files. */
/** Accumulates Entity records from level db files under a directory hierarchy. */
class RecordAccumulator {
private final LevelDbLogReader reader = new LevelDbLogReader();
/** Recursively reads all records in the directory. */
public final RecordAccumulator readDirectory(File dir) {
public final RecordAccumulator readDirectory(File dir, Predicate<File> fileMatcher) {
for (File child : dir.listFiles()) {
if (child.isDirectory()) {
readDirectory(child);
} else if (child.isFile()) {
readDirectory(child, fileMatcher);
} else if (fileMatcher.test(child)) {
try {
reader.readFrom(new FileInputStream(child));
} catch (IOException e) {