mirror of
https://github.com/google/nomulus
synced 2026-06-09 16:33:02 +00:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Rachel Guan
|
3beb207fcc | ||
 gbrodman
|
8cf88b7e18 | ||
|
gbrodman
|
6ec2e9501d | ||
 sarahcaseybot
|
6849bf6914 | ||
|
gbrodman
|
34f3823960 | ||
|
gbrodman
|
bb5d2dcf0a | ||
|
sarahcaseybot
|
6ce0211537 | ||
 Lai Jiang
|
676616a172 | ||
 Weimin Yu
|
62c556cebf |
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -79,8 +79,8 @@ PRESUBMITS = {
|
||||
r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
|
||||
("java", "js", "soy", "sql", "py", "sh", "gradle"), {
|
||||
".git", "/build/", "/generated/", "/generated_tests/",
|
||||
"node_modules/", "JUnitBackports.java", "registrar_bin.",
|
||||
"registrar_dbg.", "google-java-format-diff.py",
|
||||
"node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java",
|
||||
"registrar_bin.", "registrar_dbg.", "google-java-format-diff.py",
|
||||
"nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
|
||||
}, REQUIRED):
|
||||
"File did not include the license header.",
|
||||
|
||||
@@ -197,6 +197,7 @@ dependencies {
|
||||
compile deps['com.google.apis:google-api-services-groupssettings']
|
||||
compile deps['com.google.apis:google-api-services-monitoring']
|
||||
compile deps['com.google.apis:google-api-services-sheets']
|
||||
compile deps['com.google.apis:google-api-services-storage']
|
||||
testCompile deps['com.google.appengine:appengine-api-stubs']
|
||||
compile deps['com.google.appengine.tools:appengine-gcs-client']
|
||||
compile deps['com.google.appengine.tools:appengine-mapreduce']
|
||||
@@ -221,6 +222,8 @@ dependencies {
|
||||
gradleLint.ignore('unused-dependency') {
|
||||
compile deps['com.google.gwt:gwt-user']
|
||||
}
|
||||
compile deps['com.google.cloud:google-cloud-core']
|
||||
compile deps['com.google.cloud:google-cloud-storage']
|
||||
compile deps['com.google.http-client:google-http-client']
|
||||
compile deps['com.google.http-client:google-http-client-appengine']
|
||||
compile deps['com.google.http-client:google-http-client-jackson2']
|
||||
@@ -315,6 +318,7 @@ dependencies {
|
||||
annotationProcessor project(':processor')
|
||||
testAnnotationProcessor project(':processor')
|
||||
|
||||
testCompile deps['com.google.cloud:google-cloud-nio']
|
||||
testCompile deps['com.google.appengine:appengine-testing']
|
||||
testCompile deps['com.google.guava:guava-testlib']
|
||||
testCompile deps['com.google.monitoring-client:contrib']
|
||||
|
||||
@@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33
|
||||
com.google.dagger:dagger-spi:2.33
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotation:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_check_api:2.3.4
|
||||
com.google.errorprone:error_prone_core:2.3.4
|
||||
com.google.errorprone:error_prone_type_annotations:2.3.4
|
||||
com.google.errorprone:javac-shaded:9-dev-r4023-3
|
||||
com.google.googlejavaformat:google-java-format:1.5
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.protobuf:protobuf-java:3.4.0
|
||||
@@ -32,7 +32,7 @@ javax.inject:javax.inject:1
|
||||
javax.persistence:javax.persistence-api:2.2
|
||||
net.ltgt.gradle.incap:incap:0.2
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.checkerframework:dataflow:3.0.0
|
||||
org.checkerframework:javacutil:3.0.0
|
||||
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
|
||||
|
||||
@@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -215,7 +216,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -232,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -216,7 +217,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33
|
||||
com.google.dagger:dagger-spi:2.33
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotation:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_check_api:2.3.4
|
||||
com.google.errorprone:error_prone_core:2.3.4
|
||||
com.google.errorprone:error_prone_type_annotations:2.3.4
|
||||
com.google.errorprone:javac-shaded:9-dev-r4023-3
|
||||
com.google.googlejavaformat:google-java-format:1.5
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.protobuf:protobuf-java:3.4.0
|
||||
@@ -30,7 +30,7 @@ javax.inject:javax.inject:1
|
||||
javax.persistence:javax.persistence-api:2.2
|
||||
net.ltgt.gradle.incap:incap:0.2
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.checkerframework:dataflow:3.0.0
|
||||
org.checkerframework:javacutil:3.0.0
|
||||
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
|
||||
|
||||
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -54,12 +54,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -84,10 +84,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -98,32 +98,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -135,9 +137,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -161,7 +163,7 @@ io.github.classgraph:classgraph:4.8.102
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
@@ -199,7 +201,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
net.java.dev.jna:jna:5.5.0
|
||||
@@ -307,7 +309,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -53,12 +53,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -83,10 +83,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -97,31 +97,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -133,9 +135,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -159,7 +161,7 @@ io.github.classgraph:classgraph:4.8.102
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
@@ -194,7 +196,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
net.java.dev.jna:jna:5.5.0
|
||||
@@ -301,7 +303,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
@@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.arnx:nashorn-promise:0.1.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -320,7 +322,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
@@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.arnx:nashorn-promise:0.1.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -321,7 +323,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -18,8 +18,10 @@ import static com.google.appengine.api.ThreadManager.currentRequestThreadFactory
|
||||
import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.BUCKET_OVERRIDE_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.FROM_TIME_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.TO_TIME_PARAM;
|
||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static java.util.concurrent.Executors.newFixedThreadPool;
|
||||
@@ -32,6 +34,9 @@ import google.registry.cron.CommitLogFanoutAction;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.Parameter;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.inject.Qualifier;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -75,6 +80,12 @@ public final class BackupModule {
|
||||
return extractRequiredDatetimeParameter(req, UPPER_CHECKPOINT_TIME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(BUCKET_OVERRIDE_PARAM)
|
||||
static Optional<String> provideBucketOverride(HttpServletRequest req) {
|
||||
return extractOptionalParameter(req, BUCKET_OVERRIDE_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(FROM_TIME_PARAM)
|
||||
static DateTime provideFromTime(HttpServletRequest req) {
|
||||
@@ -92,4 +103,9 @@ public final class BackupModule {
|
||||
static ListeningExecutorService provideListeningExecutorService() {
|
||||
return listeningDecorator(newFixedThreadPool(NUM_THREADS, currentRequestThreadFactory()));
|
||||
}
|
||||
|
||||
@Provides
|
||||
static ScheduledExecutorService provideScheduledExecutorService() {
|
||||
return Executors.newSingleThreadScheduledExecutor();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,18 +28,17 @@ import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static java.nio.channels.Channels.newOutputStream;
|
||||
import static java.util.Comparator.comparingLong;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
@@ -74,7 +73,8 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
|
||||
public static final String DIFF_FILE_PREFIX = "commit_diff_until_";
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
@Inject @Config("commitLogDiffExportBatchSize") int batchSize;
|
||||
@Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime;
|
||||
@@ -102,13 +102,13 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
|
||||
logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
|
||||
// Open an output channel to GCS, wrapped in a stream for convenience.
|
||||
try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
|
||||
new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString())
|
||||
.addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString())
|
||||
.addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size()))
|
||||
.build()))) {
|
||||
try (OutputStream gcsStream =
|
||||
gcsUtils.openOutputStream(
|
||||
BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
ImmutableMap.of(
|
||||
LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(),
|
||||
UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(),
|
||||
NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) {
|
||||
// Export the upper checkpoint itself.
|
||||
serializeEntity(upperCheckpoint, gcsStream);
|
||||
// If there are no manifests to export, stop early, now that we've written out the file with
|
||||
|
||||
@@ -21,22 +21,22 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static google.registry.util.DateTimeUtils.latestOf;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.common.util.concurrent.ListeningExecutorService;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import dagger.Lazy;
|
||||
import google.registry.backup.BackupModule.Backups;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -46,33 +46,49 @@ class GcsDiffFileLister {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
@Inject @Backups ListeningExecutorService executor;
|
||||
@Inject GcsDiffFileLister() {}
|
||||
/** Timeout for retrieving per-file information from GCS. */
|
||||
private static final Duration FILE_INFO_TIMEOUT_DURATION = Duration.ofMinutes(1);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Backups Lazy<ListeningExecutorService> lazyExecutor;
|
||||
@Inject ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
@Inject
|
||||
GcsDiffFileLister() {}
|
||||
|
||||
/**
|
||||
* Traverses the sequence of diff files backwards from checkpointTime and inserts the file
|
||||
* metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
|
||||
* metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
|
||||
* more files are missing.
|
||||
*
|
||||
* @throws UncheckedExecutionException wrapping a {@link java.util.concurrent.TimeoutException} if
|
||||
* the GCS call fails to finish within one minute, or wrapping any other exception if
|
||||
* something else goes wrong.
|
||||
*/
|
||||
private boolean constructDiffSequence(
|
||||
Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
|
||||
String gcsBucket,
|
||||
Map<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo,
|
||||
DateTime fromTime,
|
||||
DateTime lastTime,
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence) {
|
||||
TreeMap<DateTime, BlobInfo> sequence) {
|
||||
DateTime checkpointTime = lastTime;
|
||||
while (isBeforeOrAt(fromTime, checkpointTime)) {
|
||||
GcsFileMetadata metadata;
|
||||
if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
|
||||
metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
|
||||
BlobInfo blobInfo;
|
||||
if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) {
|
||||
blobInfo =
|
||||
Futures.getUnchecked(
|
||||
Futures.withTimeout(
|
||||
upperBoundTimesToBlobInfo.get(checkpointTime),
|
||||
FILE_INFO_TIMEOUT_DURATION,
|
||||
scheduledExecutorService));
|
||||
} else {
|
||||
String filename = DIFF_FILE_PREFIX + checkpointTime;
|
||||
logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
|
||||
metadata = getMetadata(filename);
|
||||
blobInfo = getBlobInfo(gcsBucket, filename);
|
||||
|
||||
// If we hit a gap, quit.
|
||||
if (metadata == null) {
|
||||
if (blobInfo == null) {
|
||||
logger.atInfo().log(
|
||||
"Gap discovered in sequence terminating at %s, missing file: %s",
|
||||
sequence.lastKey(), filename);
|
||||
@@ -80,14 +96,15 @@ class GcsDiffFileLister {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sequence.put(checkpointTime, metadata);
|
||||
checkpointTime = getLowerBoundTime(metadata);
|
||||
sequence.put(checkpointTime, blobInfo);
|
||||
checkpointTime = getLowerBoundTime(blobInfo);
|
||||
}
|
||||
logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
|
||||
return true;
|
||||
}
|
||||
|
||||
ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
|
||||
ImmutableList<BlobInfo> listDiffFiles(
|
||||
String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) {
|
||||
logger.atInfo().log("Requested restore from time: %s", fromTime);
|
||||
if (toTime != null) {
|
||||
logger.atInfo().log(" Until time: %s", toTime);
|
||||
@@ -95,66 +112,69 @@ class GcsDiffFileLister {
|
||||
// List all of the diff files on GCS and build a map from each file's upper checkpoint time
|
||||
// (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
|
||||
// an upper checkpoint time > fromTime.
|
||||
TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
|
||||
= new TreeMap<>();
|
||||
Iterator<ListItem> listItems;
|
||||
TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
|
||||
ImmutableList<String> strippedFilenames;
|
||||
try {
|
||||
// TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
|
||||
listItems = gcsService.list(
|
||||
gcsBucket,
|
||||
new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
|
||||
strippedFilenames = gcsUtils.listFolderObjects(gcsBucket, DIFF_FILE_PREFIX);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
DateTime lastUpperBoundTime = START_OF_TIME;
|
||||
while (listItems.hasNext()) {
|
||||
final String filename = listItems.next().getName();
|
||||
DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToMetadata.put(upperBoundTime, executor.submit(() -> getMetadata(filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
if (upperBoundTimesToMetadata.isEmpty()) {
|
||||
logger.atInfo().log("No files found");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
// Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
|
||||
// last file that we found) and finding its previous file until we either run out of files or
|
||||
// get to one that precedes "fromTime".
|
||||
//
|
||||
// GCS file listing is eventually consistent, so it's possible that we are missing a file. The
|
||||
// metadata of a file is sufficient to identify the preceding file, so if we start from the
|
||||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet = !constructDiffSequence(
|
||||
upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
// Verify that all of the elements in the original set are represented in the sequence. If we
|
||||
// find anything that's not represented, construct a sequence for it.
|
||||
boolean checkForMoreExtraDiffs = true; // Always loop at least once.
|
||||
while (checkForMoreExtraDiffs) {
|
||||
checkForMoreExtraDiffs = false;
|
||||
for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
|
||||
if (!isInRange(key, fromTime, toTime)) {
|
||||
break;
|
||||
}
|
||||
if (!sequence.containsKey(key)) {
|
||||
constructDiffSequence(upperBoundTimesToMetadata, fromTime, key, sequence);
|
||||
checkForMoreExtraDiffs = true;
|
||||
inconsistentFileSet = true;
|
||||
break;
|
||||
TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
|
||||
try {
|
||||
for (String strippedFilename : strippedFilenames) {
|
||||
final String filename = DIFF_FILE_PREFIX + strippedFilename;
|
||||
DateTime upperBoundTime = DateTime.parse(strippedFilename);
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToBlobInfo.put(
|
||||
upperBoundTime, lazyExecutor.get().submit(() -> getBlobInfo(gcsBucket, filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (upperBoundTimesToBlobInfo.isEmpty()) {
|
||||
logger.atInfo().log("No files found");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
checkState(
|
||||
!inconsistentFileSet,
|
||||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
// Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e.
|
||||
// the last file that we found) and finding its previous file until we either run out of files
|
||||
// or get to one that precedes "fromTime".
|
||||
//
|
||||
// GCS file listing is eventually consistent, so it's possible that we are missing a file. The
|
||||
// metadata of a file is sufficient to identify the preceding file, so if we start from the
|
||||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet =
|
||||
!constructDiffSequence(
|
||||
gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
// Verify that all of the elements in the original set are represented in the sequence. If we
|
||||
// find anything that's not represented, construct a sequence for it.
|
||||
boolean checkForMoreExtraDiffs = true; // Always loop at least once.
|
||||
while (checkForMoreExtraDiffs) {
|
||||
checkForMoreExtraDiffs = false;
|
||||
for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) {
|
||||
if (!isInRange(key, fromTime, toTime)) {
|
||||
break;
|
||||
}
|
||||
if (!sequence.containsKey(key)) {
|
||||
constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
|
||||
checkForMoreExtraDiffs = true;
|
||||
inconsistentFileSet = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checkState(
|
||||
!inconsistentFileSet,
|
||||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
} finally {
|
||||
lazyExecutor.get().shutdown();
|
||||
}
|
||||
|
||||
logger.atInfo().log(
|
||||
"Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
|
||||
@@ -171,15 +191,11 @@ class GcsDiffFileLister {
|
||||
return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end));
|
||||
}
|
||||
|
||||
private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
|
||||
return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
private DateTime getLowerBoundTime(BlobInfo blobInfo) {
|
||||
return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
}
|
||||
|
||||
private GcsFileMetadata getMetadata(String filename) {
|
||||
try {
|
||||
return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
private BlobInfo getBlobInfo(String gcsBucket, String filename) {
|
||||
return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,11 +25,12 @@ import static org.joda.time.Duration.standardHours;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.Key;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
|
||||
@@ -50,7 +51,6 @@ import google.registry.util.Clock;
|
||||
import google.registry.util.RequestStatusChecker;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
@@ -69,19 +69,22 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
static final String PATH = "/_dr/task/replayCommitLogsToSql";
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final int BLOCK_SIZE =
|
||||
1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
|
||||
private static final Duration LEASE_LENGTH = standardHours(1);
|
||||
// Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
|
||||
// request timeouts
|
||||
private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject Response response;
|
||||
@Inject RequestStatusChecker requestStatusChecker;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
@Inject Clock clock;
|
||||
|
||||
@Inject
|
||||
@Config("commitLogGcsBucket")
|
||||
String gcsBucket;
|
||||
|
||||
/** If true, will exit after logging the commit log files that would otherwise be replayed. */
|
||||
@Inject
|
||||
@Parameter(DRY_RUN_PARAM)
|
||||
@@ -117,13 +120,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
}
|
||||
try {
|
||||
logger.atInfo().log("Beginning replay of commit logs.");
|
||||
ImmutableList<GcsFileMetadata> commitLogFiles = getFilesToReplay();
|
||||
ImmutableList<BlobInfo> commitLogFiles = getFilesToReplay();
|
||||
if (dryRun) {
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
ImmutableList<String> filenames =
|
||||
commitLogFiles.stream()
|
||||
.limit(10)
|
||||
.map(file -> file.getFilename().getObjectName())
|
||||
.map(file -> file.getName())
|
||||
.collect(toImmutableList());
|
||||
String dryRunMessage =
|
||||
"Running in dry-run mode; would have processed %d files. They are (limit 10):\n"
|
||||
@@ -147,22 +150,22 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
private ImmutableList<GcsFileMetadata> getFilesToReplay() {
|
||||
private ImmutableList<BlobInfo> getFilesToReplay() {
|
||||
// Start at the first millisecond we haven't seen yet
|
||||
DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
logger.atInfo().log("Starting replay from: %s.", fromTime);
|
||||
// If there's an inconsistent file set, this will throw IllegalStateException and the job
|
||||
// will try later -- this is likely because an export hasn't finished yet.
|
||||
ImmutableList<GcsFileMetadata> commitLogFiles =
|
||||
diffLister.listDiffFiles(fromTime, /* current time */ null);
|
||||
ImmutableList<BlobInfo> commitLogFiles =
|
||||
diffLister.listDiffFiles(gcsBucket, fromTime, /* current time */ null);
|
||||
logger.atInfo().log("Found %d new commit log files to process.", commitLogFiles.size());
|
||||
return commitLogFiles;
|
||||
}
|
||||
|
||||
private void replayFiles(ImmutableList<GcsFileMetadata> commitLogFiles) {
|
||||
private void replayFiles(ImmutableList<BlobInfo> commitLogFiles) {
|
||||
DateTime replayTimeoutTime = clock.nowUtc().plus(REPLAY_TIMEOUT_DURATION);
|
||||
int processedFiles = 0;
|
||||
for (GcsFileMetadata metadata : commitLogFiles) {
|
||||
for (BlobInfo metadata : commitLogFiles) {
|
||||
// One transaction per GCS file
|
||||
jpaTm().transact(() -> processFile(metadata));
|
||||
processedFiles++;
|
||||
@@ -176,31 +179,26 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
logger.atInfo().log("Replayed %d commit log files to SQL successfully.", processedFiles);
|
||||
}
|
||||
|
||||
private void processFile(GcsFileMetadata metadata) {
|
||||
private void processFile(BlobInfo metadata) {
|
||||
logger.atInfo().log(
|
||||
"Processing commit log file %s of size %d B.",
|
||||
metadata.getFilename(), metadata.getLength());
|
||||
try (InputStream input =
|
||||
Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
"Processing commit log file %s of size %d B.", metadata.getName(), metadata.getSize());
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
// Load and process the Datastore transactions one at a time
|
||||
ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
|
||||
CommitLogImports.loadEntitiesByTransaction(input);
|
||||
logger.atInfo().log(
|
||||
"Replaying %d transactions from commit log file %s.",
|
||||
allTransactions.size(), metadata.getFilename());
|
||||
allTransactions.size(), metadata.getName());
|
||||
allTransactions.forEach(this::replayTransaction);
|
||||
// if we succeeded, set the last-seen time
|
||||
DateTime checkpoint =
|
||||
DateTime.parse(
|
||||
metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
|
||||
DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
|
||||
SqlReplayCheckpoint.set(checkpoint);
|
||||
logger.atInfo().log(
|
||||
"Replayed %d transactions from commit log file %s.",
|
||||
allTransactions.size(), metadata.getFilename());
|
||||
allTransactions.size(), metadata.getName());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(
|
||||
"Errored out while replaying commit log file " + metadata.getFilename(), e);
|
||||
"Errored out while replaying commit log file " + metadata.getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import com.google.appengine.api.datastore.DatastoreService;
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.EntityTranslator;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
import com.google.common.collect.Streams;
|
||||
@@ -33,7 +33,9 @@ import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Result;
|
||||
import com.googlecode.objectify.util.ResultNow;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
@@ -46,10 +48,10 @@ import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Retrier;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
import javax.inject.Inject;
|
||||
@@ -66,43 +68,55 @@ public class RestoreCommitLogsAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
static final int BLOCK_SIZE = 1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
|
||||
public static final String PATH = "/_dr/task/restoreCommitLogs";
|
||||
static final String DRY_RUN_PARAM = "dryRun";
|
||||
static final String FROM_TIME_PARAM = "fromTime";
|
||||
static final String TO_TIME_PARAM = "toTime";
|
||||
static final String BUCKET_OVERRIDE_PARAM = "gcsBucket";
|
||||
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject @Parameter(DRY_RUN_PARAM) boolean dryRun;
|
||||
@Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime;
|
||||
@Inject @Parameter(TO_TIME_PARAM) DateTime toTime;
|
||||
|
||||
@Inject
|
||||
@Parameter(BUCKET_OVERRIDE_PARAM)
|
||||
Optional<String> gcsBucketOverride;
|
||||
|
||||
@Inject DatastoreService datastoreService;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
|
||||
@Inject
|
||||
@Config("commitLogGcsBucket")
|
||||
String defaultGcsBucket;
|
||||
|
||||
@Inject Retrier retrier;
|
||||
@Inject RestoreCommitLogsAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
checkArgument(
|
||||
RegistryEnvironment.get() == RegistryEnvironment.ALPHA
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.CRASH
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
|
||||
"DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
|
||||
!FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
|
||||
"DO NOT RUN IN PRODUCTION OR SANDBOX.");
|
||||
if (dryRun) {
|
||||
logger.atInfo().log("Running in dryRun mode");
|
||||
}
|
||||
List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
|
||||
String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
|
||||
logger.atInfo().log("Restoring from %s.", gcsBucket);
|
||||
List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
|
||||
if (diffFiles.isEmpty()) {
|
||||
logger.atInfo().log("Nothing to restore");
|
||||
return;
|
||||
}
|
||||
Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
|
||||
CommitLogCheckpoint lastCheckpoint = null;
|
||||
for (GcsFileMetadata metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
|
||||
try (InputStream input = Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
for (BlobInfo metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getName());
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
PeekingIterator<ImmutableObject> commitLogs =
|
||||
peekingIterator(createDeserializingIterator(input, true));
|
||||
lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
|
||||
|
||||
@@ -29,6 +29,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import google.registry.util.UtilsModule;
|
||||
@@ -98,6 +99,7 @@ public class Spec11Pipeline implements Serializable {
|
||||
}
|
||||
|
||||
void setupPipeline(Pipeline pipeline) {
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
PCollection<Subdomain> domains =
|
||||
options.getDatabase().equals("DATASTORE")
|
||||
? readFromBigQuery(options, pipeline)
|
||||
|
||||
@@ -416,17 +416,6 @@ public final class RegistryConfig {
|
||||
return Optional.ofNullable(config.cloudDns.servicePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns size of Google Cloud Storage client connection buffer in bytes.
|
||||
*
|
||||
* @see google.registry.gcs.GcsUtils
|
||||
*/
|
||||
@Provides
|
||||
@Config("gcsBufferSize")
|
||||
public static int provideGcsBufferSize() {
|
||||
return 1024 * 1024;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the email address of the admin account on the G Suite app used to perform
|
||||
* administrative actions.
|
||||
@@ -1288,6 +1277,18 @@ public final class RegistryConfig {
|
||||
return config.sslCertificateValidation.minimumRsaKeyLength;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningEmailBodyText")
|
||||
public static String provideExpirationWarningEmailBodyText(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningEmailBodyText;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningEmailSubjectText")
|
||||
public static String provideExpirationWarningEmailSubjectText(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningEmailSubjectText;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("allowedEcdsaCurves")
|
||||
public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
|
||||
|
||||
@@ -230,5 +230,7 @@ public class RegistryConfigSettings {
|
||||
public int expirationWarningDays;
|
||||
public int minimumRsaKeyLength;
|
||||
public Set<String> allowedEcdsaCurves;
|
||||
public String expirationWarningEmailBodyText;
|
||||
public String expirationWarningEmailSubjectText;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -452,6 +452,13 @@ sslCertificateValidation:
|
||||
# The number of days before a certificate expires that indicates the
|
||||
# certificate is nearing expiration and warnings should be sent.
|
||||
expirationWarningDays: 30
|
||||
# Text for expiring certificate notification email subject.
|
||||
expirationWarningEmailSubjectText: Certificate Expring Within 30 Days.
|
||||
# Text for expiring certificate notification email body that accepts 3 parameters:
|
||||
# registrar name, certificate type, and expiration date, respectively.
|
||||
expirationWarningEmailBodyText: |
|
||||
Hello Registrar %s,
|
||||
The %s certificate is expiring on %s.
|
||||
# The minimum number of bits an RSA key must contain.
|
||||
minimumRsaKeyLength: 2048
|
||||
# The ECDSA curves that are allowed for public keys.
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.base.Verify.verifyNotNull;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
@@ -24,11 +23,10 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Suppliers;
|
||||
@@ -79,9 +77,9 @@ public class ExportDomainListsAction implements Runnable {
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject DriveConnection driveConnection;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("domainListsGcsBucket") String gcsBucket;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject ExportDomainListsAction() {}
|
||||
|
||||
@Override
|
||||
@@ -95,7 +93,7 @@ public class ExportDomainListsAction implements Runnable {
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(clock.nowUtc(), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsUtils),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
@@ -134,7 +132,7 @@ public class ExportDomainListsAction implements Runnable {
|
||||
String domainsList = Joiner.on("\n").join(domains);
|
||||
logger.atInfo().log(
|
||||
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsUtils);
|
||||
exportToDrive(tld, domainsList, driveConnection);
|
||||
});
|
||||
}
|
||||
@@ -168,11 +166,9 @@ public class ExportDomainListsAction implements Runnable {
|
||||
}
|
||||
|
||||
protected static boolean exportToGcs(
|
||||
String tld, String domains, String gcsBucket, int gcsBufferSize) {
|
||||
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
String tld, String domains, String gcsBucket, GcsUtils gcsUtils) {
|
||||
BlobId blobId = BlobId.of(gcsBucket, tld + ".txt");
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (Throwable e) {
|
||||
@@ -214,7 +210,7 @@ public class ExportDomainListsAction implements Runnable {
|
||||
Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection());
|
||||
|
||||
private final String gcsBucket;
|
||||
private final int gcsBufferSize;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
/**
|
||||
* Non-serializable {@link DriveConnection} that will be created when an instance of {@link
|
||||
@@ -224,9 +220,9 @@ public class ExportDomainListsAction implements Runnable {
|
||||
*/
|
||||
private transient DriveConnection driveConnection;
|
||||
|
||||
public ExportDomainListsReducer(String gcsBucket, int gcsBufferSize) {
|
||||
public ExportDomainListsReducer(String gcsBucket, GcsUtils gcsUtils) {
|
||||
this.gcsBucket = gcsBucket;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@@ -240,7 +236,7 @@ public class ExportDomainListsAction implements Runnable {
|
||||
ImmutableList<String> domains = ImmutableList.sortedCopyOf(() -> fqdns);
|
||||
String domainsList = Joiner.on('\n').join(domains);
|
||||
logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize)) {
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsUtils)) {
|
||||
getContext().incrementCounter("domain lists successful written out to GCS");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to GCS");
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.gcs;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
|
||||
/** Dagger module for {@link GcsService}. */
|
||||
@Module
|
||||
public final class GcsServiceModule {
|
||||
|
||||
private static final GcsService gcsService =
|
||||
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
|
||||
|
||||
@Provides
|
||||
static GcsService provideGcsService() {
|
||||
return gcsService;
|
||||
}
|
||||
}
|
||||
@@ -14,30 +14,39 @@
|
||||
|
||||
package google.registry.gcs;
|
||||
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterables.getLast;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.cloud.storage.Blob;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.Storage;
|
||||
import com.google.cloud.storage.Storage.BlobListOption;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.CredentialModule.DefaultCredential;
|
||||
import google.registry.util.GoogleCredentialsBundle;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.io.Serializable;
|
||||
import java.nio.channels.Channels;
|
||||
import javax.annotation.CheckReturnValue;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Utilities for working with Google Cloud Storage. */
|
||||
public class GcsUtils {
|
||||
/**
|
||||
* Utilities for working with Google Cloud Storage.
|
||||
*
|
||||
* <p>It is {@link Serializable} so that it can be used in MapReduce or Beam.
|
||||
*/
|
||||
public class GcsUtils implements Serializable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@@ -48,30 +57,65 @@ public class GcsUtils {
|
||||
.put("json", MediaType.JSON_UTF_8)
|
||||
.build();
|
||||
|
||||
private final GcsService gcsService;
|
||||
private final int bufferSize;
|
||||
private final StorageOptions storageOptions;
|
||||
|
||||
@Inject
|
||||
public GcsUtils(GcsService gcsService, @Config("gcsBufferSize") int bufferSize) {
|
||||
this.gcsService = gcsService;
|
||||
this.bufferSize = bufferSize;
|
||||
private Storage storage() {
|
||||
return storageOptions.getService();
|
||||
}
|
||||
|
||||
/** Opens a GCS file for reading as an {@link InputStream} with prefetching. */
|
||||
@Inject
|
||||
public GcsUtils(@DefaultCredential GoogleCredentialsBundle credentialsBundle) {
|
||||
this(
|
||||
StorageOptions.newBuilder()
|
||||
.setCredentials(credentialsBundle.getGoogleCredentials())
|
||||
.build());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public GcsUtils(StorageOptions storageOptions) {
|
||||
this.storageOptions = storageOptions;
|
||||
}
|
||||
|
||||
/** Opens a GCS file for reading as an {@link InputStream}. */
|
||||
@CheckReturnValue
|
||||
public InputStream openInputStream(GcsFilename filename) {
|
||||
return Channels.newInputStream(gcsService.openPrefetchingReadChannel(filename, 0, bufferSize));
|
||||
public InputStream openInputStream(BlobId blobId) {
|
||||
return Channels.newInputStream(storage().reader(blobId));
|
||||
}
|
||||
|
||||
/** Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files. */
|
||||
@CheckReturnValue
|
||||
public OutputStream openOutputStream(GcsFilename filename) throws IOException {
|
||||
return Channels.newOutputStream(gcsService.createOrReplace(filename, getOptions(filename)));
|
||||
public OutputStream openOutputStream(BlobId blobId) {
|
||||
return Channels.newOutputStream(storage().writer(createBlobInfo(blobId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files and setting
|
||||
* the given metadata.
|
||||
*/
|
||||
@CheckReturnValue
|
||||
public OutputStream openOutputStream(BlobId blobId, ImmutableMap<String, String> metadata) {
|
||||
return Channels.newOutputStream(
|
||||
storage().writer(BlobInfo.newBuilder(blobId).setMetadata(metadata).build()));
|
||||
}
|
||||
|
||||
/** Creates a GCS file with the given byte contents, overwriting existing files. */
|
||||
public void createFromBytes(GcsFilename filename, byte[] bytes) throws IOException {
|
||||
gcsService.createOrReplace(filename, getOptions(filename), ByteBuffer.wrap(bytes));
|
||||
public void createFromBytes(BlobId blobId, byte[] bytes) throws StorageException {
|
||||
createFromBytes(createBlobInfo(blobId), bytes);
|
||||
}
|
||||
|
||||
/** Creates a GCS file with the given byte contents and metadata, overwriting existing files. */
|
||||
public void createFromBytes(BlobInfo blobInfo, byte[] bytes) throws StorageException {
|
||||
storage().create(blobInfo, bytes);
|
||||
}
|
||||
|
||||
/** Read the content of the given GCS file and return it in a byte array. */
|
||||
public byte[] readBytesFrom(BlobId blobId) throws StorageException {
|
||||
return storage().readAllBytes(blobId);
|
||||
}
|
||||
|
||||
/** Delete the given GCS file. */
|
||||
public void delete(BlobId blobId) throws StorageException {
|
||||
storage().delete(blobId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -86,37 +130,55 @@ public class GcsUtils {
|
||||
*/
|
||||
public ImmutableList<String> listFolderObjects(String bucketName, String prefix)
|
||||
throws IOException {
|
||||
ListResult result =
|
||||
gcsService.list(bucketName, new ListOptions.Builder().setPrefix(prefix).build());
|
||||
final ImmutableList.Builder<String> builder = new ImmutableList.Builder<>();
|
||||
result.forEachRemaining(
|
||||
listItem -> {
|
||||
if (!listItem.isDirectory()) {
|
||||
builder.add(listItem.getName().replaceFirst(prefix, ""));
|
||||
}
|
||||
});
|
||||
return builder.build();
|
||||
return Streams.stream(storage().list(bucketName, BlobListOption.prefix(prefix)).iterateAll())
|
||||
.map(blob -> blob.getName().substring(prefix.length()))
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
|
||||
/** Returns {@code true} if a file exists and is non-empty on Google Cloud Storage. */
|
||||
public boolean existsAndNotEmpty(GcsFilename file) {
|
||||
GcsFileMetadata metadata;
|
||||
public boolean existsAndNotEmpty(BlobId blobId) {
|
||||
try {
|
||||
metadata = gcsService.getMetadata(file);
|
||||
} catch (IOException e) {
|
||||
Blob blob = storage().get(blobId);
|
||||
return blob != null && blob.getSize() > 0;
|
||||
} catch (StorageException e) {
|
||||
logger.atWarning().withCause(e).log("Failed to check if GCS file exists");
|
||||
return false;
|
||||
}
|
||||
return metadata != null && metadata.getLength() > 0;
|
||||
}
|
||||
|
||||
/** Determines most appropriate {@link GcsFileOptions} based on filename extension. */
|
||||
private static GcsFileOptions getOptions(GcsFilename filename) {
|
||||
GcsFileOptions.Builder builder = new GcsFileOptions.Builder().cacheControl("no-cache");
|
||||
MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(filename.getObjectName())));
|
||||
/** Returns the user defined metadata of a GCS file if the file exists, or an empty map. */
|
||||
public ImmutableMap<String, String> getMetadata(BlobId blobId) throws StorageException {
|
||||
Blob blob = storage().get(blobId);
|
||||
return blob == null ? ImmutableMap.of() : ImmutableMap.copyOf(blob.getMetadata());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the {@link BlobInfo} of the given GCS file.
|
||||
*
|
||||
* <p>Note that a {@link Blob} is returned, but on the {@link BlobInfo} part of it is usable.
|
||||
*/
|
||||
public BlobInfo getBlobInfo(BlobId blobId) throws StorageException {
|
||||
return storage().get(blobId);
|
||||
}
|
||||
|
||||
/** Determines most appropriate {@link BlobInfo} based on filename extension. */
|
||||
private static BlobInfo createBlobInfo(BlobId blobId) {
|
||||
BlobInfo.Builder builder = BlobInfo.newBuilder(blobId).setCacheControl("no-cache");
|
||||
MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(blobId.getName())));
|
||||
if (mediaType != null) {
|
||||
builder = builder.mimeType(mediaType.type());
|
||||
builder = builder.setContentType(mediaType.toString());
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// These two methods are needed to check whether serialization is done correctly in tests.
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
return obj instanceof GcsUtils && ((GcsUtils) obj).storageOptions.equals(storageOptions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return storageOptions.hashCode();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,16 +164,12 @@ public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends Dom
|
||||
|
||||
/** Gets the names of the tlds that reference this list. */
|
||||
public final ImmutableSet<String> getReferencingTlds() {
|
||||
Key<? extends BaseDomainLabelList<?, ?>> key = Key.create(this);
|
||||
return getTlds()
|
||||
.stream()
|
||||
.filter((tld) -> refersToKey(Registry.get(tld), key))
|
||||
return getTlds().stream()
|
||||
.filter((tld) -> refersToList(Registry.get(tld), name))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
|
||||
// TODO(b/193043636): Refactor this class to no longer use key references
|
||||
protected abstract boolean refersToKey(
|
||||
Registry registry, Key<? extends BaseDomainLabelList<?, ?>> key);
|
||||
protected abstract boolean refersToList(Registry registry, String name);
|
||||
|
||||
protected static <R> Optional<R> getFromCache(String listName, LoadingCache<String, R> cache) {
|
||||
try {
|
||||
|
||||
@@ -282,8 +282,8 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean refersToKey(Registry registry, Key<? extends BaseDomainLabelList<?, ?>> key) {
|
||||
return Objects.equals(registry.getPremiumListName().orElse(null), key.getName());
|
||||
public boolean refersToList(Registry registry, String name) {
|
||||
return Objects.equals(registry.getPremiumListName().orElse(null), name);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -33,7 +33,6 @@ import com.google.common.cache.LoadingCache;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Embed;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Mapify;
|
||||
@@ -185,8 +184,8 @@ public final class ReservedList
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean refersToKey(Registry registry, Key<? extends BaseDomainLabelList<?, ?>> key) {
|
||||
return registry.getReservedListNames().contains(key.getName());
|
||||
protected boolean refersToList(Registry registry, String name) {
|
||||
return registry.getReservedListNames().contains(name);
|
||||
}
|
||||
|
||||
/** Determines whether the ReservedList is in use on any Registry */
|
||||
|
||||
@@ -26,7 +26,6 @@ import google.registry.export.datastore.DatastoreAdminModule;
|
||||
import google.registry.export.sheet.SheetsServiceModule;
|
||||
import google.registry.flows.ServerTridProviderModule;
|
||||
import google.registry.flows.custom.CustomLogicFactoryModule;
|
||||
import google.registry.gcs.GcsServiceModule;
|
||||
import google.registry.groups.DirectoryModule;
|
||||
import google.registry.groups.GroupsModule;
|
||||
import google.registry.groups.GroupssettingsModule;
|
||||
@@ -64,7 +63,6 @@ import javax.inject.Singleton;
|
||||
DirectoryModule.class,
|
||||
DummyKeyringModule.class,
|
||||
DriveModule.class,
|
||||
GcsServiceModule.class,
|
||||
GroupsModule.class,
|
||||
GroupssettingsModule.class,
|
||||
JSchModule.class,
|
||||
|
||||
@@ -22,7 +22,6 @@ import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.export.DriveModule;
|
||||
import google.registry.flows.ServerTridProviderModule;
|
||||
import google.registry.flows.custom.CustomLogicFactoryModule;
|
||||
import google.registry.gcs.GcsServiceModule;
|
||||
import google.registry.groups.DirectoryModule;
|
||||
import google.registry.groups.GroupsModule;
|
||||
import google.registry.groups.GroupssettingsModule;
|
||||
@@ -54,7 +53,6 @@ import javax.inject.Singleton;
|
||||
DirectoryModule.class,
|
||||
DummyKeyringModule.class,
|
||||
DriveModule.class,
|
||||
GcsServiceModule.class,
|
||||
GroupsModule.class,
|
||||
GroupssettingsModule.class,
|
||||
Jackson2Module.class,
|
||||
|
||||
+31
-6
@@ -16,17 +16,22 @@ package google.registry.persistence.transaction;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.api.utils.SystemProperty;
|
||||
import com.google.appengine.api.utils.SystemProperty.Environment.Value;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Suppliers;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
|
||||
import google.registry.model.ofy.DatastoreTransactionManager;
|
||||
import google.registry.persistence.DaggerPersistenceComponent;
|
||||
import google.registry.tools.RegistryToolEnvironment;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Factory class to create {@link TransactionManager} instance. */
|
||||
// TODO: Rename this to PersistenceFactory and move to persistence package.
|
||||
@@ -34,7 +39,8 @@ public class TransactionManagerFactory {
|
||||
|
||||
private static final DatastoreTransactionManager ofyTm = createTransactionManager();
|
||||
|
||||
@NonFinalForTesting private static TransactionManager tm = ofyTm;
|
||||
/** Optional override to manually set the transaction manager per-test. */
|
||||
private static Optional<TransactionManager> tmForTest = Optional.empty();
|
||||
|
||||
/** Supplier for jpaTm so that it is initialized only once, upon first usage. */
|
||||
@NonFinalForTesting
|
||||
@@ -70,9 +76,19 @@ public class TransactionManagerFactory {
|
||||
return SystemProperty.environment.value() == Value.Production;
|
||||
}
|
||||
|
||||
/** Returns {@link TransactionManager} instance. */
|
||||
/**
|
||||
* Returns the {@link TransactionManager} instance.
|
||||
*
|
||||
* <p>Returns the {@link JpaTransactionManager} or {@link DatastoreTransactionManager} based on
|
||||
* the migration schedule or the manually specified per-test transaction manager.
|
||||
*/
|
||||
public static TransactionManager tm() {
|
||||
return tm;
|
||||
if (tmForTest.isPresent()) {
|
||||
return tmForTest.get();
|
||||
}
|
||||
PrimaryDatabase primaryDatabase =
|
||||
DatabaseMigrationStateSchedule.getValueAtTime(DateTime.now(UTC)).getPrimaryDatabase();
|
||||
return primaryDatabase.equals(PrimaryDatabase.DATASTORE) ? ofyTm() : jpaTm();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -111,9 +127,18 @@ public class TransactionManagerFactory {
|
||||
jpaTm = Suppliers.memoize(jpaTmSupplier::get);
|
||||
}
|
||||
|
||||
/** Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}. */
|
||||
/**
|
||||
* Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}.
|
||||
*
|
||||
* <p>Used when overriding the per-test transaction manager for dual-database tests.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
public static void setTm(TransactionManager newTm) {
|
||||
tm = newTm;
|
||||
public static void setTmForTest(TransactionManager newTm) {
|
||||
tmForTest = Optional.of(newTm);
|
||||
}
|
||||
|
||||
/** Resets the overridden transaction manager post-test. */
|
||||
public static void removeTmOverrideForTest() {
|
||||
tmForTest = Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ package google.registry.rde;
|
||||
import static google.registry.model.rde.RdeMode.THIN;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
@@ -85,10 +85,10 @@ public final class BrdaCopyAction implements Runnable {
|
||||
|
||||
private void copyAsRyde() throws IOException {
|
||||
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
|
||||
GcsFilename xmlFilename = new GcsFilename(stagingBucket, prefix + ".xml.ghostryde");
|
||||
GcsFilename xmlLengthFilename = new GcsFilename(stagingBucket, prefix + ".xml.length");
|
||||
GcsFilename rydeFile = new GcsFilename(brdaBucket, prefix + ".ryde");
|
||||
GcsFilename sigFile = new GcsFilename(brdaBucket, prefix + ".sig");
|
||||
BlobId xmlFilename = BlobId.of(stagingBucket, prefix + ".xml.ghostryde");
|
||||
BlobId xmlLengthFilename = BlobId.of(stagingBucket, prefix + ".xml.length");
|
||||
BlobId rydeFile = BlobId.of(brdaBucket, prefix + ".ryde");
|
||||
BlobId sigFile = BlobId.of(brdaBucket, prefix + ".sig");
|
||||
|
||||
long xmlLength = readXmlLength(xmlLengthFilename);
|
||||
|
||||
@@ -107,7 +107,7 @@ public final class BrdaCopyAction implements Runnable {
|
||||
}
|
||||
|
||||
/** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */
|
||||
private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException {
|
||||
private long readXmlLength(BlobId xmlLengthFilename) throws IOException {
|
||||
try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) {
|
||||
return Ghostryde.readLength(input);
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import static google.registry.persistence.transaction.TransactionManagerUtil.tra
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
@@ -97,7 +97,7 @@ public final class RdeReportAction implements Runnable, EscrowTask {
|
||||
.orElseThrow(
|
||||
() -> new IllegalStateException("RdeRevision was not set on generated deposit"));
|
||||
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
|
||||
GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
|
||||
BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde");
|
||||
verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
|
||||
reporter.send(readReportFromGcs(reportFilename));
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
@@ -106,7 +106,7 @@ public final class RdeReportAction implements Runnable, EscrowTask {
|
||||
}
|
||||
|
||||
/** Reads and decrypts the XML file from cloud storage. */
|
||||
private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException {
|
||||
private byte[] readReportFromGcs(BlobId reportFilename) throws IOException {
|
||||
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename);
|
||||
InputStream ghostrydeDecoder = Ghostryde.decoder(gcsInput, stagingDecryptionKey)) {
|
||||
return ByteStreams.toByteArray(ghostrydeDecoder);
|
||||
|
||||
@@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.Multimaps;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.mapreduce.inputs.NullInput;
|
||||
@@ -206,6 +207,7 @@ public final class RdeStagingAction implements Runnable {
|
||||
@Inject PendingDepositChecker pendingDepositChecker;
|
||||
@Inject RdeStagingReducer.Factory reducerFactory;
|
||||
@Inject Response response;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject @Config("transactionCooldown") Duration transactionCooldown;
|
||||
@Inject @Parameter(RdeModule.PARAM_MANUAL) boolean manual;
|
||||
@@ -234,7 +236,7 @@ public final class RdeStagingAction implements Runnable {
|
||||
}
|
||||
ValidationMode validationMode = lenient ? LENIENT : STRICT;
|
||||
RdeStagingMapper mapper = new RdeStagingMapper(validationMode, pendings);
|
||||
RdeStagingReducer reducer = reducerFactory.create(validationMode);
|
||||
RdeStagingReducer reducer = reducerFactory.create(validationMode, gcsUtils);
|
||||
|
||||
mrRunner
|
||||
.setJobName("Stage escrow deposits for all TLDs")
|
||||
|
||||
@@ -16,7 +16,6 @@ package google.registry.rde;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
|
||||
@@ -24,10 +23,9 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
@@ -69,27 +67,27 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
|
||||
private final TaskQueueUtils taskQueueUtils;
|
||||
private final LockHandler lockHandler;
|
||||
private final int gcsBufferSize;
|
||||
private final String bucket;
|
||||
private final Duration lockTimeout;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final RdeMarshaller marshaller;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
RdeStagingReducer(
|
||||
TaskQueueUtils taskQueueUtils,
|
||||
LockHandler lockHandler,
|
||||
int gcsBufferSize,
|
||||
String bucket,
|
||||
Duration lockTimeout,
|
||||
byte[] stagingKeyBytes,
|
||||
ValidationMode validationMode) {
|
||||
ValidationMode validationMode,
|
||||
GcsUtils gcsUtils) {
|
||||
this.taskQueueUtils = taskQueueUtils;
|
||||
this.lockHandler = lockHandler;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.bucket = bucket;
|
||||
this.lockTimeout = lockTimeout;
|
||||
this.stagingKeyBytes = stagingKeyBytes;
|
||||
this.marshaller = new RdeMarshaller(validationMode);
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -113,8 +111,6 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
|
||||
// Construct things that Dagger would inject if this wasn't serialized.
|
||||
PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
RdeCounter counter = new RdeCounter();
|
||||
|
||||
// Determine some basic things about the deposit.
|
||||
@@ -130,14 +126,14 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
|
||||
prefix = "manual/" + key.directoryWithTrailingSlash() + prefix;
|
||||
}
|
||||
GcsFilename xmlFilename = new GcsFilename(bucket, prefix + ".xml.ghostryde");
|
||||
BlobId xmlFilename = BlobId.of(bucket, prefix + ".xml.ghostryde");
|
||||
// This file will contain the byte length (ASCII) of the raw unencrypted XML.
|
||||
//
|
||||
// This is necessary because RdeUploadAction creates a tar file which requires that the length
|
||||
// be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
|
||||
// length, so we just save it separately.
|
||||
GcsFilename xmlLengthFilename = new GcsFilename(bucket, prefix + ".xml.length");
|
||||
GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
|
||||
BlobId xmlLengthFilename = BlobId.of(bucket, prefix + ".xml.length");
|
||||
BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde");
|
||||
|
||||
// These variables will be populated as we write the deposit XML and used for other files.
|
||||
boolean failed = false;
|
||||
@@ -146,8 +142,8 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
// Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
|
||||
|
||||
logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = cloudStorage.openOutputStream(xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
|
||||
Writer output = new OutputStreamWriter(ghostrydeEncoder, UTF_8)) {
|
||||
|
||||
@@ -194,7 +190,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
// This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
|
||||
if (mode == RdeMode.FULL) {
|
||||
logger.atInfo().log("Writing %s", reportFilename);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(reportFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
|
||||
counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
|
||||
} catch (IOException | XmlException e) {
|
||||
@@ -249,22 +245,21 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
static class Factory {
|
||||
@Inject TaskQueueUtils taskQueueUtils;
|
||||
@Inject LockHandler lockHandler;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject @Config("rdeBucket") String bucket;
|
||||
@Inject @Config("rdeStagingLockTimeout") Duration lockTimeout;
|
||||
@Inject @KeyModule.Key("rdeStagingEncryptionKey") byte[] stagingKeyBytes;
|
||||
|
||||
@Inject Factory() {}
|
||||
|
||||
RdeStagingReducer create(ValidationMode validationMode) {
|
||||
RdeStagingReducer create(ValidationMode validationMode, GcsUtils gcsUtils) {
|
||||
return new RdeStagingReducer(
|
||||
taskQueueUtils,
|
||||
lockHandler,
|
||||
gcsBufferSize,
|
||||
bucket,
|
||||
lockTimeout,
|
||||
stagingKeyBytes,
|
||||
validationMode);
|
||||
validationMode,
|
||||
gcsUtils);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static java.util.Arrays.asList;
|
||||
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.io.ByteStreams;
|
||||
@@ -165,9 +165,9 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
||||
.orElseThrow(
|
||||
() -> new IllegalStateException("RdeRevision was not set on generated deposit"));
|
||||
final String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
|
||||
final GcsFilename xmlFilename = new GcsFilename(bucket, name + ".xml.ghostryde");
|
||||
final GcsFilename xmlLengthFilename = new GcsFilename(bucket, name + ".xml.length");
|
||||
GcsFilename reportFilename = new GcsFilename(bucket, name + "-report.xml.ghostryde");
|
||||
final BlobId xmlFilename = BlobId.of(bucket, name + ".xml.ghostryde");
|
||||
final BlobId xmlLengthFilename = BlobId.of(bucket, name + ".xml.length");
|
||||
BlobId reportFilename = BlobId.of(bucket, name + "-report.xml.ghostryde");
|
||||
verifyFileExists(xmlFilename);
|
||||
verifyFileExists(xmlLengthFilename);
|
||||
verifyFileExists(reportFilename);
|
||||
@@ -187,29 +187,30 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs a blocking upload of a cloud storage XML file to escrow provider, converting
|
||||
* it to the RyDE format along the way by applying tar+compress+encrypt+sign, and saving the
|
||||
* created RyDE file on GCS for future reference.
|
||||
* Performs a blocking upload of a cloud storage XML file to escrow provider, converting it to the
|
||||
* RyDE format along the way by applying tar+compress+encrypt+sign, and saving the created RyDE
|
||||
* file on GCS for future reference.
|
||||
*
|
||||
* <p>This is done by layering a bunch of {@link java.io.FilterOutputStream FilterOutputStreams}
|
||||
* on top of each other in reverse order that turn XML bytes into a RyDE file while
|
||||
* simultaneously uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to
|
||||
* blocking-copy bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline.
|
||||
* on top of each other in reverse order that turn XML bytes into a RyDE file while simultaneously
|
||||
* uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to blocking-copy
|
||||
* bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline.
|
||||
*
|
||||
* <p>In pseudo-shell, the whole process looks like the following:
|
||||
*
|
||||
* <pre> {@code
|
||||
* gcs read $xmlFile \ # Get GhostRyDE from cloud storage.
|
||||
* | decrypt | decompress \ # Convert it to XML.
|
||||
* | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
|
||||
* | tee gs://bucket/$rydeFilename.ryde \ # Save a copy of the RyDE file to GCS.
|
||||
* | sftp put $dstUrl/$rydeFilename.ryde \ # Upload to SFTP server.
|
||||
* && sftp put $dstUrl/$rydeFilename.sig </tmp/sig \ # Upload detached signature.
|
||||
* && cat /tmp/sig > gs://bucket/$rydeFilename.sig # Save a copy of signature to GCS.
|
||||
* }</pre>
|
||||
* <pre>{@code
|
||||
* gcs read $xmlFile \ # Get GhostRyDE from cloud storage.
|
||||
* | decrypt | decompress \ # Convert it to XML.
|
||||
* | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
|
||||
* | tee gs://bucket/$rydeFilename.ryde \ # Save a copy of the RyDE file to GCS.
|
||||
* | sftp put $dstUrl/$rydeFilename.ryde \ # Upload to SFTP server.
|
||||
* && sftp put $dstUrl/$rydeFilename.sig </tmp/sig \ # Upload detached signature.
|
||||
* && cat /tmp/sig > gs://bucket/$rydeFilename.sig # Save a copy of signature to GCS.
|
||||
*
|
||||
* }</pre>
|
||||
*/
|
||||
@VisibleForTesting
|
||||
protected void upload(GcsFilename xmlFile, long xmlLength, DateTime watermark, String name)
|
||||
protected void upload(BlobId xmlFile, long xmlLength, DateTime watermark, String name)
|
||||
throws Exception {
|
||||
logger.atInfo().log("Uploading XML file '%s' to remote path '%s'.", xmlFile, uploadUrl);
|
||||
try (InputStream gcsInput = gcsUtils.openInputStream(xmlFile);
|
||||
@@ -218,7 +219,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
||||
JSchSftpChannel ftpChan = session.openSftpChannel()) {
|
||||
ByteArrayOutputStream sigOut = new ByteArrayOutputStream();
|
||||
String rydeFilename = name + ".ryde";
|
||||
GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename);
|
||||
BlobId rydeGcsFilename = BlobId.of(bucket, rydeFilename);
|
||||
try (OutputStream ftpOutput =
|
||||
ftpChan.get().put(rydeFilename, sftpProgressMonitor, OVERWRITE);
|
||||
OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename);
|
||||
@@ -234,7 +235,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
||||
}
|
||||
String sigFilename = name + ".sig";
|
||||
byte[] signature = sigOut.toByteArray();
|
||||
gcsUtils.createFromBytes(new GcsFilename(bucket, sigFilename), signature);
|
||||
gcsUtils.createFromBytes(BlobId.of(bucket, sigFilename), signature);
|
||||
ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename);
|
||||
logger.atInfo().log("uploaded %,d bytes: %s", signature.length, sigFilename);
|
||||
}
|
||||
@@ -242,13 +243,13 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
||||
}
|
||||
|
||||
/** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */
|
||||
private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException {
|
||||
private long readXmlLength(BlobId xmlLengthFilename) throws IOException {
|
||||
try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) {
|
||||
return Ghostryde.readLength(input);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyFileExists(GcsFilename filename) {
|
||||
private void verifyFileExists(BlobId filename) {
|
||||
verify(gcsUtils.existsAndNotEmpty(filename), "Missing file: %s", filename);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ package google.registry.reporting.billing;
|
||||
import static com.google.common.base.Throwables.getRootCause;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.io.CharStreams;
|
||||
import com.google.common.net.MediaType;
|
||||
@@ -72,8 +72,7 @@ public class BillingEmailUtils {
|
||||
void emailOverallInvoice() {
|
||||
try {
|
||||
String invoiceFile = String.format("%s-%s.csv", invoiceFilePrefix, yearMonth);
|
||||
GcsFilename invoiceFilename =
|
||||
new GcsFilename(billingBucket, invoiceDirectoryPrefix + invoiceFile);
|
||||
BlobId invoiceFilename = BlobId.of(billingBucket, invoiceDirectoryPrefix + invoiceFile);
|
||||
try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) {
|
||||
emailService.sendEmail(
|
||||
EmailMessage.newBuilder()
|
||||
|
||||
@@ -19,7 +19,7 @@ import static google.registry.request.Action.Method.POST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
@@ -120,7 +120,7 @@ public final class CopyDetailReportsAction implements Runnable {
|
||||
() -> {
|
||||
try (InputStream input =
|
||||
gcsUtils.openInputStream(
|
||||
new GcsFilename(billingBucket, invoiceDirectoryPrefix + detailReportName))) {
|
||||
BlobId.of(billingBucket, invoiceDirectoryPrefix + detailReportName))) {
|
||||
driveConnection.createOrUpdateFile(
|
||||
detailReportName,
|
||||
MediaType.CSV_UTF_8,
|
||||
|
||||
@@ -21,7 +21,7 @@ import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.ImmutableCollection;
|
||||
@@ -260,7 +260,7 @@ public class IcannReportingStager {
|
||||
Ascii.toLowerCase(reportType.toString()),
|
||||
DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, reportFilename);
|
||||
gcsUtils.createFromBytes(gcsFilename, reportBytes);
|
||||
logger.atInfo().log("Wrote %d bytes to file location %s", reportBytes.length, gcsFilename);
|
||||
return reportFilename;
|
||||
@@ -269,7 +269,7 @@ public class IcannReportingStager {
|
||||
/** Creates and stores a manifest file on GCS, indicating which reports were generated. */
|
||||
void createAndUploadManifest(String subdir, ImmutableList<String> filenames) throws IOException {
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, MANIFEST_FILE_NAME);
|
||||
StringBuilder manifestString = new StringBuilder();
|
||||
filenames.forEach((filename) -> manifestString.append(filename).append("\n"));
|
||||
gcsUtils.createFromBytes(gcsFilename, manifestString.toString().getBytes(UTF_8));
|
||||
|
||||
@@ -20,7 +20,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Maps;
|
||||
@@ -147,7 +147,7 @@ public final class IcannReportingUploadAction implements Runnable {
|
||||
cursorTimeMinusMonth.getYear(), cursorTimeMinusMonth.getMonthOfYear());
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, reportSubdir);
|
||||
String filename = getFileName(cursorType, cursorTime, tldStr);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, filename);
|
||||
logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname);
|
||||
// Check that the report exists
|
||||
try {
|
||||
@@ -298,18 +298,18 @@ public final class IcannReportingUploadAction implements Runnable {
|
||||
emailService.sendEmail(EmailMessage.create(subject, body, recipient, sender));
|
||||
}
|
||||
|
||||
private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException {
|
||||
private byte[] readBytesFromGcs(BlobId reportFilename) throws IOException {
|
||||
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) {
|
||||
return ByteStreams.toByteArray(gcsInput);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyFileExists(GcsFilename gcsFilename) {
|
||||
private void verifyFileExists(BlobId gcsFilename) {
|
||||
checkArgument(
|
||||
gcsUtils.existsAndNotEmpty(gcsFilename),
|
||||
"Object %s in bucket %s not found",
|
||||
gcsFilename.getObjectName(),
|
||||
gcsFilename.getBucketName());
|
||||
gcsFilename.getName(),
|
||||
gcsFilename.getBucket());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+5
-5
@@ -17,7 +17,7 @@ package google.registry.reporting.spec11;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
@@ -62,7 +62,7 @@ public class Spec11RegistrarThreatMatchesParser {
|
||||
}
|
||||
|
||||
/** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */
|
||||
public ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
|
||||
public ImmutableSet<RegistrarThreatMatches> getFromFile(BlobId spec11ReportFilename)
|
||||
throws IOException {
|
||||
if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) {
|
||||
return ImmutableSet.of();
|
||||
@@ -81,7 +81,7 @@ public class Spec11RegistrarThreatMatchesParser {
|
||||
|
||||
public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
|
||||
LocalDate yesterday = date.minusDays(1);
|
||||
GcsFilename gcsFilename = getGcsFilename(yesterday);
|
||||
BlobId gcsFilename = getGcsFilename(yesterday);
|
||||
if (gcsUtils.existsAndNotEmpty(gcsFilename)) {
|
||||
return Optional.of(yesterday);
|
||||
}
|
||||
@@ -98,8 +98,8 @@ public class Spec11RegistrarThreatMatchesParser {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
private GcsFilename getGcsFilename(LocalDate localDate) {
|
||||
return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
|
||||
private BlobId getGcsFilename(LocalDate localDate) {
|
||||
return BlobId.of(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
|
||||
}
|
||||
|
||||
private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
package google.registry.tools.server;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterators.filter;
|
||||
import static com.google.common.io.BaseEncoding.base16;
|
||||
@@ -25,11 +24,10 @@ import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
@@ -98,13 +96,14 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject JsonActionRunner jsonActionRunner;
|
||||
@Inject @Config("zoneFilesBucket") String bucket;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject @Config("commitLogDatastoreRetention") Duration datastoreRetention;
|
||||
@Inject @Config("dnsDefaultATtl") Duration dnsDefaultATtl;
|
||||
@SuppressWarnings("DurationVariableWithUnits") // false-positive Error Prone check
|
||||
@Inject @Config("dnsDefaultNsTtl") Duration dnsDefaultNsTtl;
|
||||
@Inject @Config("dnsDefaultDsTtl") Duration dnsDefaultDsTtl;
|
||||
@Inject Clock clock;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject GenerateZoneFilesAction() {}
|
||||
|
||||
@Override
|
||||
@@ -140,7 +139,7 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
|
||||
.runMapreduce(
|
||||
new GenerateBindFileMapper(
|
||||
tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl),
|
||||
new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize),
|
||||
new GenerateBindFileReducer(bucket, exportTime, gcsUtils),
|
||||
ImmutableList.of(new NullInput<>(), createEntityInput(DomainBase.class)))
|
||||
.getLinkToMapreduceConsole();
|
||||
ImmutableList<String> filenames =
|
||||
@@ -236,22 +235,19 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
|
||||
|
||||
private final String bucket;
|
||||
private final DateTime exportTime;
|
||||
private final int gcsBufferSize;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
GenerateBindFileReducer(String bucket, DateTime exportTime, int gcsBufferSize) {
|
||||
GenerateBindFileReducer(String bucket, DateTime exportTime, GcsUtils gcsUtils) {
|
||||
this.bucket = bucket;
|
||||
this.exportTime = exportTime;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reduce(String tld, ReducerInput<String> stanzas) {
|
||||
String stanzaCounter = tld + " stanzas";
|
||||
GcsFilename filename =
|
||||
new GcsFilename(bucket, String.format(FILENAME_FORMAT, tld, exportTime));
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
BlobId filename = BlobId.of(bucket, String.format(FILENAME_FORMAT, tld, exportTime));
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(filename);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8);
|
||||
PrintWriter writer = new PrintWriter(osWriter)) {
|
||||
writer.printf(HEADER_FORMAT, tld);
|
||||
|
||||
@@ -24,18 +24,17 @@ import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.GcsTestingUtils;
|
||||
import google.registry.testing.TestObject;
|
||||
import java.util.List;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -53,8 +52,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
.withOfyTestEntities(TestObject.class)
|
||||
.build();
|
||||
|
||||
/** Local GCS service available for testing. */
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private final DateTime now = DateTime.now(UTC);
|
||||
private final DateTime oneMinuteAgo = now.minusMinutes(1);
|
||||
@@ -63,7 +61,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
task.gcsService = gcsService;
|
||||
task.gcsUtils = gcsUtils;
|
||||
task.gcsBucket = "gcs bucket";
|
||||
task.batchSize = 5;
|
||||
}
|
||||
@@ -84,10 +82,11 @@ public class ExportCommitLogDiffActionTest {
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
@@ -95,8 +94,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"0");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported).containsExactly(upperCheckpoint);
|
||||
}
|
||||
|
||||
@@ -139,10 +137,11 @@ public class ExportCommitLogDiffActionTest {
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
@@ -150,8 +149,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"4");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in time order, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(2, now.minusDays(1).minusMillis(1));
|
||||
@@ -191,10 +189,11 @@ public class ExportCommitLogDiffActionTest {
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
@@ -202,8 +201,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"4");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo);
|
||||
@@ -246,10 +244,11 @@ public class ExportCommitLogDiffActionTest {
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
@@ -257,8 +256,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"6");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo);
|
||||
@@ -301,10 +299,11 @@ public class ExportCommitLogDiffActionTest {
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
@@ -312,8 +311,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"0");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
// We expect no manifests or mutations, only the upper checkpoint.
|
||||
assertThat(exported).containsExactly(upperCheckpoint);
|
||||
}
|
||||
@@ -359,11 +357,11 @@ public class ExportCommitLogDiffActionTest {
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename))
|
||||
.isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
@@ -371,8 +369,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"6");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in time order, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(3, oneMinuteAgo.minusDays(2));
|
||||
@@ -415,10 +412,11 @@ public class ExportCommitLogDiffActionTest {
|
||||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
START_OF_TIME.toString(),
|
||||
@@ -426,8 +424,7 @@ public class ExportCommitLogDiffActionTest {
|
||||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"3");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, START_OF_TIME.plusMillis(1));
|
||||
|
||||
@@ -16,32 +16,28 @@ package google.registry.backup;
|
||||
|
||||
import static com.google.common.collect.Iterables.transform;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static java.lang.reflect.Proxy.newProxyInstance;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.flogger.LoggerConfig;
|
||||
import com.google.common.testing.TestLogHandler;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.InvocationHandler;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.logging.LogRecord;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -55,7 +51,7 @@ public class GcsDiffFileListerTest {
|
||||
|
||||
private final DateTime now = DateTime.now(UTC);
|
||||
private final GcsDiffFileLister diffLister = new GcsDiffFileLister();
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final TestLogHandler logHandler = new TestLogHandler();
|
||||
|
||||
@RegisterExtension
|
||||
@@ -64,39 +60,32 @@ public class GcsDiffFileListerTest {
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
diffLister.gcsService = gcsService;
|
||||
diffLister.gcsBucket = GCS_BUCKET;
|
||||
diffLister.executor = newDirectExecutorService();
|
||||
diffLister.gcsUtils = gcsUtils;
|
||||
diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
|
||||
diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
gcsService.createOrReplace(
|
||||
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(i)),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(i + 1).toString())
|
||||
.build(),
|
||||
ByteBuffer.wrap(new byte[]{1, 2, 3}));
|
||||
addGcsFile(i, i + 1);
|
||||
}
|
||||
LoggerConfig.getConfig(GcsDiffFileLister.class).addHandler(logHandler);
|
||||
}
|
||||
|
||||
private Iterable<DateTime> extractTimesFromDiffFiles(List<GcsFileMetadata> diffFiles) {
|
||||
private Iterable<DateTime> extractTimesFromDiffFiles(ImmutableList<BlobInfo> diffFiles) {
|
||||
return transform(
|
||||
diffFiles,
|
||||
metadata ->
|
||||
DateTime.parse(
|
||||
metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length())));
|
||||
blobInfo -> DateTime.parse(blobInfo.getName().substring(DIFF_FILE_PREFIX.length())));
|
||||
}
|
||||
|
||||
private Iterable<DateTime> listDiffFiles(DateTime fromTime, DateTime toTime) {
|
||||
return extractTimesFromDiffFiles(diffLister.listDiffFiles(fromTime, toTime));
|
||||
return extractTimesFromDiffFiles(diffLister.listDiffFiles(GCS_BUCKET, fromTime, toTime));
|
||||
}
|
||||
|
||||
private void addGcsFile(int fileAge, int prevAge) throws IOException {
|
||||
gcsService.createOrReplace(
|
||||
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge)),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString())
|
||||
.build(),
|
||||
ByteBuffer.wrap(new byte[]{1, 2, 3}));
|
||||
BlobInfo blobInfo =
|
||||
BlobInfo.newBuilder(BlobId.of(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge)))
|
||||
.setMetadata(
|
||||
ImmutableMap.of(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString()))
|
||||
.build();
|
||||
gcsUtils.createFromBytes(blobInfo, new byte[] {1, 2, 3});
|
||||
}
|
||||
|
||||
private void assertLogContains(String message) {
|
||||
@@ -115,38 +104,11 @@ public class GcsDiffFileListerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void testList_patchesHoles() {
|
||||
// Fake out the GCS list() method to return only the first and last file.
|
||||
// We can't use Mockito.spy() because GcsService's impl is final.
|
||||
diffLister.gcsService = (GcsService) newProxyInstance(
|
||||
GcsService.class.getClassLoader(),
|
||||
new Class<?>[] {GcsService.class},
|
||||
new InvocationHandler() {
|
||||
@Override
|
||||
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
|
||||
if (method.getName().equals("list")) {
|
||||
// ListResult is an incredibly annoying thing to construct. It needs to be fed from a
|
||||
// Callable that returns Iterators, each representing a batch of results.
|
||||
return new ListResult(new Callable<Iterator<ListItem>>() {
|
||||
boolean called = false;
|
||||
|
||||
@Override
|
||||
public Iterator<ListItem> call() {
|
||||
try {
|
||||
return called ? null : Iterators.forArray(
|
||||
new ListItem.Builder()
|
||||
.setName(DIFF_FILE_PREFIX + now)
|
||||
.build(),
|
||||
new ListItem.Builder()
|
||||
.setName(DIFF_FILE_PREFIX + now.minusMinutes(4))
|
||||
.build());
|
||||
} finally {
|
||||
called = true;
|
||||
}
|
||||
}});
|
||||
}
|
||||
return method.invoke(gcsService, args);
|
||||
}});
|
||||
void testList_patchesHoles() throws Exception {
|
||||
GcsUtils mockGcsUtils = mock(GcsUtils.class);
|
||||
diffLister.gcsUtils = spy(gcsUtils);
|
||||
when(mockGcsUtils.listFolderObjects(anyString(), anyString()))
|
||||
.thenReturn(ImmutableList.of(now.toString(), now.minusMinutes(4).toString()));
|
||||
DateTime fromTime = now.minusMinutes(4).minusSeconds(1);
|
||||
// Request all files with checkpoint > fromTime.
|
||||
assertThat(listDiffFiles(fromTime, null))
|
||||
|
||||
@@ -16,8 +16,6 @@ package google.registry.backup;
|
||||
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
|
||||
import static google.registry.backup.RestoreCommitLogsActionTest.GCS_BUCKET;
|
||||
import static google.registry.backup.RestoreCommitLogsActionTest.createCheckpoint;
|
||||
import static google.registry.backup.RestoreCommitLogsActionTest.saveDiffFile;
|
||||
import static google.registry.backup.RestoreCommitLogsActionTest.saveDiffFileNotToRestore;
|
||||
@@ -38,14 +36,15 @@ import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import com.google.common.truth.Truth8;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
@@ -73,8 +72,10 @@ import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.TestObject;
|
||||
import google.registry.util.RequestStatusChecker;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -111,7 +112,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
.build();
|
||||
|
||||
/** Local GCS service. */
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private final ReplayCommitLogsToSqlAction action = new ReplayCommitLogsToSqlAction();
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
@@ -124,14 +125,15 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
action.gcsService = gcsService;
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.response = response;
|
||||
action.requestStatusChecker = requestStatusChecker;
|
||||
action.clock = fakeClock;
|
||||
action.gcsBucket = "gcs bucket";
|
||||
action.diffLister = new GcsDiffFileLister();
|
||||
action.diffLister.gcsService = gcsService;
|
||||
action.diffLister.gcsBucket = GCS_BUCKET;
|
||||
action.diffLister.executor = newDirectExecutorService();
|
||||
action.diffLister.gcsUtils = gcsUtils;
|
||||
action.diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
|
||||
action.diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
|
||||
ofyTm()
|
||||
.transact(
|
||||
() ->
|
||||
@@ -145,6 +147,11 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
TestObject.beforeSqlDeleteCallCount = 0;
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
ofyTm().transact(() -> DatabaseMigrationStateSchedule.set(DEFAULT_TRANSITION_MAP.toValueMap()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReplay_multipleDiffFiles() throws Exception {
|
||||
jpaTm()
|
||||
@@ -166,9 +173,9 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
|
||||
Key<CommitLogManifest> manifest2Key =
|
||||
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -183,7 +190,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
|
||||
CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -201,8 +208,8 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
void testReplay_noManifests() throws Exception {
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFile(gcsService, createCheckpoint(now.minusMillis(2)));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
saveDiffFile(gcsUtils, createCheckpoint(now.minusMillis(2)));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMillis(1)));
|
||||
runAndAssertSuccess(now.minusMillis(1));
|
||||
assertExpectedIds("previous to keep");
|
||||
@@ -215,10 +222,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(1);
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
||||
@@ -238,10 +245,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(1);
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
||||
@@ -259,10 +266,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
jpaTm().insertWithoutBackup(TestObject.create("previous to keep"));
|
||||
jpaTm().insertWithoutBackup(TestObject.create("previous to delete"));
|
||||
});
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -277,10 +284,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> jpaTm().put(TestObject.create("existing", "a")));
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1).minusMillis(1));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMillis(1)),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
|
||||
@@ -295,10 +302,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
void testReplay_deleteMissingEntity() throws Exception {
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> jpaTm().put(TestObject.create("previous to keep", "a")));
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1).minusMillis(1));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMillis(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -333,7 +340,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
TransactionManagerFactory.setJpaTm(() -> spy);
|
||||
// Save in the commit logs the domain and contact (in that order) and the token deletion
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(toDelete))),
|
||||
@@ -376,7 +383,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
TransactionManagerFactory.setJpaTm(() -> spy);
|
||||
// Save two commits -- the deletion, then the new version of the contact
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1).plusMillis(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(contact))),
|
||||
@@ -416,7 +423,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
() -> {
|
||||
try {
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(1), ImmutableSet.of()),
|
||||
@@ -445,7 +452,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
// Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant)
|
||||
Key<ClaimsList> claimsListKey = Key.create(ClaimsList.class, 1L);
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -492,7 +499,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")));
|
||||
@@ -505,7 +512,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
||||
@@ -14,11 +14,9 @@
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.collect.Iterables.transform;
|
||||
import static com.google.common.collect.Maps.toMap;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.BackupUtils.serializeEntity;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
@@ -28,15 +26,17 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.api.datastore.DatastoreServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.io.Resources;
|
||||
import com.google.common.primitives.Longs;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
@@ -51,10 +51,11 @@ import google.registry.testing.TestObject;
|
||||
import google.registry.util.Retrier;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -67,7 +68,7 @@ public class RestoreCommitLogsActionTest {
|
||||
|
||||
private final DateTime now = DateTime.now(UTC);
|
||||
private final RestoreCommitLogsAction action = new RestoreCommitLogsAction();
|
||||
private final GcsService gcsService = createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
@RegisterExtension
|
||||
public final AppEngineExtension appEngine =
|
||||
@@ -78,15 +79,17 @@ public class RestoreCommitLogsActionTest {
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
action.gcsService = gcsService;
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.dryRun = false;
|
||||
action.datastoreService = DatastoreServiceFactory.getDatastoreService();
|
||||
action.fromTime = now.minusMillis(1);
|
||||
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 1);
|
||||
action.defaultGcsBucket = GCS_BUCKET;
|
||||
action.gcsBucketOverride = Optional.empty();
|
||||
action.diffLister = new GcsDiffFileLister();
|
||||
action.diffLister.gcsService = gcsService;
|
||||
action.diffLister.gcsBucket = GCS_BUCKET;
|
||||
action.diffLister.executor = newDirectExecutorService();
|
||||
action.diffLister.gcsUtils = gcsUtils;
|
||||
action.diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
|
||||
action.diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -107,10 +110,10 @@ public class RestoreCommitLogsActionTest {
|
||||
CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
|
||||
Key<CommitLogManifest> manifest2Key =
|
||||
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
Iterable<ImmutableObject> file1CommitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -126,7 +129,7 @@ public class RestoreCommitLogsActionTest {
|
||||
CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
|
||||
Iterable<ImmutableObject> file2CommitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -147,8 +150,8 @@ public class RestoreCommitLogsActionTest {
|
||||
@Test
|
||||
void testRestore_noManifests() throws Exception {
|
||||
auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(gcsService, createCheckpoint(now));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(gcsUtils, createCheckpoint(now));
|
||||
action.run();
|
||||
auditedOfy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep");
|
||||
@@ -162,10 +165,10 @@ public class RestoreCommitLogsActionTest {
|
||||
auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(1);
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
||||
@@ -184,10 +187,10 @@ public class RestoreCommitLogsActionTest {
|
||||
.saveWithoutBackup()
|
||||
.entities(TestObject.create("previous to keep"), TestObject.create("previous to delete"))
|
||||
.now();
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -205,12 +208,10 @@ public class RestoreCommitLogsActionTest {
|
||||
@Test
|
||||
void testRestore_manifestWithNoMutationsOrDeletions() throws Exception {
|
||||
auditedOfy().saveWithoutBackup().entities(TestObject.create("previous to keep")).now();
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null));
|
||||
gcsUtils, createCheckpoint(now), CommitLogManifest.create(getBucketKey(1), now, null));
|
||||
action.run();
|
||||
auditedOfy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep");
|
||||
@@ -223,10 +224,10 @@ public class RestoreCommitLogsActionTest {
|
||||
void testRestore_mutateExistingEntity() throws Exception {
|
||||
auditedOfy().saveWithoutBackup().entity(TestObject.create("existing", "a")).now();
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
|
||||
@@ -243,10 +244,10 @@ public class RestoreCommitLogsActionTest {
|
||||
@Test
|
||||
void testRestore_deleteMissingEntity() throws Exception {
|
||||
auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep", "a")).now();
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
@@ -268,7 +269,7 @@ public class RestoreCommitLogsActionTest {
|
||||
// imported, in particular, the domain's 'registrant' key can be used by Objectify to load the
|
||||
// contact.
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
Resources.toByteArray(Resources.getResource("google/registry/backup/commitlog.data")),
|
||||
now);
|
||||
action.run();
|
||||
@@ -284,18 +285,18 @@ public class RestoreCommitLogsActionTest {
|
||||
return CommitLogCheckpoint.create(now, toMap(getBucketIds(), x -> now));
|
||||
}
|
||||
|
||||
static void saveDiffFile(GcsService gcsService, byte[] rawBytes, DateTime timestamp)
|
||||
static void saveDiffFile(GcsUtils gcsUtils, byte[] rawBytes, DateTime timestamp)
|
||||
throws IOException {
|
||||
gcsService.createOrReplace(
|
||||
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString())
|
||||
.build(),
|
||||
ByteBuffer.wrap(rawBytes));
|
||||
BlobInfo blobInfo =
|
||||
BlobInfo.newBuilder(BlobId.of(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp))
|
||||
.setMetadata(
|
||||
ImmutableMap.of(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString()))
|
||||
.build();
|
||||
gcsUtils.createFromBytes(blobInfo, rawBytes);
|
||||
}
|
||||
|
||||
static Iterable<ImmutableObject> saveDiffFile(
|
||||
GcsService gcsService, CommitLogCheckpoint checkpoint, ImmutableObject... entities)
|
||||
GcsUtils gcsUtils, CommitLogCheckpoint checkpoint, ImmutableObject... entities)
|
||||
throws IOException {
|
||||
DateTime now = checkpoint.getCheckpointTime();
|
||||
List<ImmutableObject> allEntities = Lists.asList(checkpoint, entities);
|
||||
@@ -303,13 +304,13 @@ public class RestoreCommitLogsActionTest {
|
||||
for (ImmutableObject entity : allEntities) {
|
||||
serializeEntity(entity, output);
|
||||
}
|
||||
saveDiffFile(gcsService, output.toByteArray(), now);
|
||||
saveDiffFile(gcsUtils, output.toByteArray(), now);
|
||||
return allEntities;
|
||||
}
|
||||
|
||||
static void saveDiffFileNotToRestore(GcsService gcsService, DateTime now) throws Exception {
|
||||
static void saveDiffFileNotToRestore(GcsUtils gcsUtils, DateTime now) throws Exception {
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null),
|
||||
CommitLogMutation.create(
|
||||
|
||||
@@ -23,7 +23,6 @@ import static google.registry.model.common.Cursor.CursorType.RDE_STAGING;
|
||||
import static google.registry.model.rde.RdeMode.FULL;
|
||||
import static google.registry.model.rde.RdeMode.THIN;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.setTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.rde.RdeResourceType.CONTACT;
|
||||
import static google.registry.rde.RdeResourceType.DOMAIN;
|
||||
@@ -50,7 +49,7 @@ import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.Registrar.State;
|
||||
import google.registry.persistence.transaction.JpaTestRules;
|
||||
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationTestExtension;
|
||||
import google.registry.persistence.transaction.TransactionManager;
|
||||
import google.registry.persistence.transaction.TransactionManagerFactory;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.PendingDeposit;
|
||||
import google.registry.rde.RdeResourceType;
|
||||
@@ -117,12 +116,9 @@ public class RdePipelineTest {
|
||||
|
||||
private RdePipeline rdePipeline;
|
||||
|
||||
private TransactionManager originalTm;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
originalTm = tm();
|
||||
setTm(jpaTm());
|
||||
TransactionManagerFactory.setTmForTest(jpaTm());
|
||||
loadInitialData();
|
||||
|
||||
// Two real registrars have been created by loadInitialData(), named "New Registrar" and "The
|
||||
@@ -169,7 +165,7 @@ public class RdePipelineTest {
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
setTm(originalTm);
|
||||
TransactionManagerFactory.removeTmOverrideForTest();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -18,8 +18,8 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.setTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.removeTmOverrideForTest;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.setTmForTest;
|
||||
import static google.registry.testing.AppEngineExtension.makeRegistrar1;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveContact;
|
||||
@@ -49,7 +49,6 @@ import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
|
||||
import google.registry.model.reporting.Spec11ThreatMatchDao;
|
||||
import google.registry.persistence.transaction.JpaTestRules;
|
||||
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationTestExtension;
|
||||
import google.registry.persistence.transaction.TransactionManager;
|
||||
import google.registry.testing.DatastoreEntityExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeSleeper;
|
||||
@@ -230,8 +229,7 @@ class Spec11PipelineTest {
|
||||
}
|
||||
|
||||
private void setupCloudSql() {
|
||||
TransactionManager originalTm = tm();
|
||||
setTm(jpaTm());
|
||||
setTmForTest(jpaTm());
|
||||
persistNewRegistrar("TheRegistrar");
|
||||
persistNewRegistrar("NewRegistrar");
|
||||
Registrar registrar1 =
|
||||
@@ -271,7 +269,7 @@ class Spec11PipelineTest {
|
||||
persistResource(createDomain("no-email.com", "2A4BA9BBC-COM", registrar2, contact2));
|
||||
persistResource(
|
||||
createDomain("anti-anti-anti-virus.dev", "555666888-DEV", registrar3, contact3));
|
||||
setTm(originalTm);
|
||||
removeTmOverrideForTest();
|
||||
}
|
||||
|
||||
private void verifySaveToGcs() throws Exception {
|
||||
|
||||
@@ -14,14 +14,12 @@
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_FILENAME;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
@@ -29,12 +27,13 @@ import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.export.ExportDomainListsAction.ExportDomainListsReducer;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
@@ -46,7 +45,6 @@ import google.registry.testing.InjectExtension;
|
||||
import google.registry.testing.TestOfyAndSql;
|
||||
import google.registry.testing.TestOfyOnly;
|
||||
import google.registry.testing.mapreduce.MapreduceTestCase;
|
||||
import java.io.FileNotFoundException;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Order;
|
||||
@@ -57,7 +55,7 @@ import org.mockito.ArgumentCaptor;
|
||||
@DualDatabaseTest
|
||||
class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAction> {
|
||||
|
||||
private GcsService gcsService;
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private DriveConnection driveConnection = mock(DriveConnection.class);
|
||||
private ArgumentCaptor<byte[]> bytesExportedToDrive = ArgumentCaptor.forClass(byte[].class);
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
@@ -81,10 +79,9 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
||||
action.mrRunner = makeDefaultRunner();
|
||||
action.response = response;
|
||||
action.gcsBucket = "outputbucket";
|
||||
action.gcsBufferSize = 500;
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.clock = clock;
|
||||
action.driveConnection = driveConnection;
|
||||
gcsService = createGcsService();
|
||||
}
|
||||
|
||||
private void runAction() throws Exception {
|
||||
@@ -117,8 +114,8 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
||||
persistActiveDomain("rudnitzky.tld");
|
||||
persistDeletedDomain("mortuary.tld", DateTime.parse("2001-03-14T10:11:12Z"));
|
||||
runAction();
|
||||
GcsFilename existingFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, existingFile), UTF_8);
|
||||
BlobId existingFile = BlobId.of("outputbucket", "tld.txt");
|
||||
String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8);
|
||||
// Check that it only contains the active domains, not the dead one.
|
||||
assertThat(tlds).isEqualTo("onetwo.tld\nrudnitzky.tld");
|
||||
verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld");
|
||||
@@ -131,17 +128,15 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
||||
persistActiveDomain("rudnitzky.tld");
|
||||
persistActiveDomain("wontgo.testtld");
|
||||
runAction();
|
||||
GcsFilename existingFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, existingFile), UTF_8).trim();
|
||||
BlobId existingFile = BlobId.of("outputbucket", "tld.txt");
|
||||
String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8).trim();
|
||||
// Check that it only contains the domains on the real TLD, and not the test one.
|
||||
assertThat(tlds).isEqualTo("onetwo.tld\nrudnitzky.tld");
|
||||
// Make sure that the test TLD file wasn't written out.
|
||||
GcsFilename nonexistentFile = new GcsFilename("outputbucket", "testtld.txt");
|
||||
assertThrows(FileNotFoundException.class, () -> readGcsFile(gcsService, nonexistentFile));
|
||||
ListResult ls = gcsService.list("outputbucket", ListOptions.DEFAULT);
|
||||
assertThat(ls.next().getName()).isEqualTo("tld.txt");
|
||||
// Make sure that no other files were written out.
|
||||
assertThat(ls.hasNext()).isFalse();
|
||||
BlobId nonexistentFile = BlobId.of("outputbucket", "testtld.txt");
|
||||
assertThrows(StorageException.class, () -> gcsUtils.readBytesFrom(nonexistentFile));
|
||||
ImmutableList<String> ls = gcsUtils.listFolderObjects("outputbucket", "");
|
||||
assertThat(ls).containsExactly("tld.txt");
|
||||
verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld");
|
||||
verifyNoMoreInteractions(driveConnection);
|
||||
}
|
||||
@@ -160,14 +155,14 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
||||
persistActiveDomain("buddy.tldtwo");
|
||||
persistActiveDomain("cupid.tldthree");
|
||||
runAction();
|
||||
GcsFilename firstTldFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, firstTldFile), UTF_8).trim();
|
||||
BlobId firstTldFile = BlobId.of("outputbucket", "tld.txt");
|
||||
String tlds = new String(gcsUtils.readBytesFrom(firstTldFile), UTF_8).trim();
|
||||
assertThat(tlds).isEqualTo("dasher.tld\nprancer.tld");
|
||||
GcsFilename secondTldFile = new GcsFilename("outputbucket", "tldtwo.txt");
|
||||
String moreTlds = new String(readGcsFile(gcsService, secondTldFile), UTF_8).trim();
|
||||
BlobId secondTldFile = BlobId.of("outputbucket", "tldtwo.txt");
|
||||
String moreTlds = new String(gcsUtils.readBytesFrom(secondTldFile), UTF_8).trim();
|
||||
assertThat(moreTlds).isEqualTo("buddy.tldtwo\nrudolph.tldtwo\nsanta.tldtwo");
|
||||
GcsFilename thirdTldFile = new GcsFilename("outputbucket", "tldthree.txt");
|
||||
String evenMoreTlds = new String(readGcsFile(gcsService, thirdTldFile), UTF_8).trim();
|
||||
BlobId thirdTldFile = BlobId.of("outputbucket", "tldthree.txt");
|
||||
String evenMoreTlds = new String(gcsUtils.readBytesFrom(thirdTldFile), UTF_8).trim();
|
||||
assertThat(evenMoreTlds).isEqualTo("cupid.tldthree");
|
||||
verifyExportedToDrive("brouhaha", "dasher.tld\nprancer.tld");
|
||||
verifyExportedToDrive("hooray", "buddy.tldtwo\nrudolph.tldtwo\nsanta.tldtwo");
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.gcs;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.io.OutputStream;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Unit tests for {@link GcsUtilsTest}. */
|
||||
class GcsUtilsTest {
|
||||
|
||||
private GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private String bucket = "my-bucket";
|
||||
private String filename = "my-file";
|
||||
private BlobId blobId = BlobId.of(bucket, filename);
|
||||
private ImmutableMap<String, String> metadata = ImmutableMap.of("key1", "val1", "Key2", "val2");
|
||||
private final byte[] bytes = new byte[] {'a', 'b', 'c'};
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {}
|
||||
|
||||
@Test
|
||||
void testSerialization_testStorage() throws Exception {
|
||||
assertThat(deserialize(serialize(gcsUtils))).isEqualTo(gcsUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSerialization_realStorage() throws Exception {
|
||||
gcsUtils = new GcsUtils(StorageOptions.getDefaultInstance());
|
||||
assertThat(deserialize(serialize(gcsUtils))).isEqualTo(gcsUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testStreams() throws Exception {
|
||||
try (OutputStream os = gcsUtils.openOutputStream(blobId, metadata)) {
|
||||
os.write(bytes);
|
||||
os.flush();
|
||||
}
|
||||
assertThat(ByteStreams.toByteArray(gcsUtils.openInputStream(blobId))).isEqualTo(bytes);
|
||||
assertThat(gcsUtils.getBlobInfo(blobId).getMetadata()).containsExactlyEntriesIn(metadata);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCreateListReadDelete() throws Exception {
|
||||
gcsUtils.createFromBytes(BlobInfo.newBuilder(blobId).setMetadata(metadata).build(), bytes);
|
||||
assertThat(gcsUtils.existsAndNotEmpty(blobId)).isTrue();
|
||||
assertThat(gcsUtils.listFolderObjects(bucket, "")).containsExactly("my-file");
|
||||
assertThat(gcsUtils.getBlobInfo(blobId).getMetadata()).isEqualTo(metadata);
|
||||
assertThat(gcsUtils.readBytesFrom(blobId)).isEqualTo(bytes);
|
||||
gcsUtils.delete(blobId);
|
||||
assertThat(gcsUtils.existsAndNotEmpty(blobId)).isFalse();
|
||||
assertThat(gcsUtils.listFolderObjects(bucket, "")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetContentType() {
|
||||
blobId = BlobId.of(bucket, "something.json");
|
||||
gcsUtils.createFromBytes(blobId, bytes);
|
||||
assertThat(gcsUtils.getBlobInfo(blobId).getContentType())
|
||||
.isEqualTo(MediaType.JSON_UTF_8.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testList() throws Exception {
|
||||
ImmutableList<BlobId> blobIds =
|
||||
ImmutableList.of(
|
||||
BlobId.of(bucket, "a/b/xyz.txt"),
|
||||
BlobId.of(bucket, "a/cde.exe"),
|
||||
BlobId.of(bucket, "fgh.jpg"),
|
||||
BlobId.of(bucket, "c/ijk.mp4"));
|
||||
|
||||
for (BlobId blobId : blobIds) {
|
||||
gcsUtils.createFromBytes(blobId, bytes);
|
||||
}
|
||||
assertThat(gcsUtils.listFolderObjects(bucket, ""))
|
||||
.containsExactly("a/b/xyz.txt", "a/cde.exe", "fgh.jpg", "c/ijk.mp4");
|
||||
assertThat(gcsUtils.listFolderObjects(bucket, "a/")).containsExactly("b/xyz.txt", "cde.exe");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testEmptyFile() {
|
||||
gcsUtils.createFromBytes(blobId, new byte[] {});
|
||||
assertThat(gcsUtils.existsAndNotEmpty(blobId)).isFalse();
|
||||
}
|
||||
|
||||
private static byte[] serialize(Object object) throws IOException {
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
ObjectOutputStream oos = new ObjectOutputStream(baos);
|
||||
oos.writeObject(object);
|
||||
oos.flush();
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
private static Object deserialize(byte[] bytes) throws IOException, ClassNotFoundException {
|
||||
try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
|
||||
return ois.readObject();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
/*
|
||||
* Copyright 2016 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package google.registry.gcs.backport;
|
||||
|
||||
import com.google.api.client.util.DateTime;
|
||||
import com.google.api.services.storage.model.Bucket;
|
||||
import com.google.api.services.storage.model.ServiceAccount;
|
||||
import com.google.api.services.storage.model.StorageObject;
|
||||
import com.google.cloud.Tuple;
|
||||
import com.google.cloud.storage.Storage;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.cloud.storage.spi.v1.StorageRpc;
|
||||
import com.google.cloud.storage.testing.StorageRpcTestBase;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
/**
|
||||
* A bare-bones in-memory implementation of StorageRpc, meant for testing.
|
||||
*
|
||||
* <p>This class is <i>not</i> thread-safe. It's also (currently) limited in the following ways:
|
||||
*
|
||||
* <ul>
|
||||
* <li>Supported
|
||||
* <ul>
|
||||
* <li>object create
|
||||
* <li>object get
|
||||
* <li>object delete
|
||||
* <li>list the contents of a bucket
|
||||
* <li>generations
|
||||
* </ul>
|
||||
* <li>Unsupported
|
||||
* <ul>
|
||||
* <li>bucket create
|
||||
* <li>bucket get
|
||||
* <li>bucket delete
|
||||
* <li>list all buckets
|
||||
* <li>file attributes
|
||||
* <li>patch
|
||||
* <li>continueRewrite
|
||||
* <li>createBatch
|
||||
* <li>checksums, etags
|
||||
* <li>IAM operations
|
||||
* <li>BucketLock operations
|
||||
* <li>HMAC key operations
|
||||
* </ul>
|
||||
* </ul>
|
||||
*/
|
||||
@NotThreadSafe
|
||||
class FakeStorageRpc extends StorageRpcTestBase {
|
||||
|
||||
private static final SimpleDateFormat RFC_3339_FORMATTER =
|
||||
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
|
||||
|
||||
// fullname -> metadata
|
||||
Map<String, StorageObject> metadata = new HashMap<>();
|
||||
// fullname -> contents
|
||||
Map<String, byte[]> contents = new HashMap<>();
|
||||
// fullname -> future contents that will be visible on close.
|
||||
Map<String, byte[]> futureContents = new HashMap<>();
|
||||
|
||||
private final boolean throwIfOption;
|
||||
|
||||
/** @param throwIfOption if true, we throw when given any option */
|
||||
public FakeStorageRpc(boolean throwIfOption) {
|
||||
this.throwIfOption = throwIfOption;
|
||||
}
|
||||
|
||||
// remove all files
|
||||
void reset() {
|
||||
metadata = new HashMap<>();
|
||||
contents = new HashMap<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject create(StorageObject object, InputStream content, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
String key = fullname(object);
|
||||
object.setUpdated(now());
|
||||
metadata.put(key, object);
|
||||
try {
|
||||
contents.put(key, com.google.common.io.ByteStreams.toByteArray(content));
|
||||
} catch (IOException e) {
|
||||
throw new StorageException(e);
|
||||
}
|
||||
// TODO: crc, etc
|
||||
return object;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
String delimiter = null;
|
||||
String preprefix = "";
|
||||
String pageToken = null;
|
||||
long maxResults = Long.MAX_VALUE;
|
||||
for (Map.Entry<Option, ?> e : options.entrySet()) {
|
||||
switch (e.getKey()) {
|
||||
case PAGE_TOKEN:
|
||||
pageToken = (String) e.getValue();
|
||||
break;
|
||||
case PREFIX:
|
||||
preprefix = (String) e.getValue();
|
||||
if (preprefix.startsWith("/")) {
|
||||
preprefix = preprefix.substring(1);
|
||||
}
|
||||
break;
|
||||
case DELIMITER:
|
||||
delimiter = (String) e.getValue();
|
||||
break;
|
||||
case FIELDS:
|
||||
// ignore and return all the fields
|
||||
break;
|
||||
case MAX_RESULTS:
|
||||
maxResults = (Long) e.getValue();
|
||||
break;
|
||||
case USER_PROJECT:
|
||||
// prevent unsupported operation
|
||||
break;
|
||||
default:
|
||||
throw new UnsupportedOperationException("Unknown option: " + e.getKey());
|
||||
}
|
||||
}
|
||||
final String prefix = preprefix;
|
||||
|
||||
List<StorageObject> values = new ArrayList<>();
|
||||
Map<String, StorageObject> folders = new HashMap<>();
|
||||
for (StorageObject so : metadata.values()) {
|
||||
if (!so.getBucket().equals(bucket) || !so.getName().startsWith(prefix)) {
|
||||
continue;
|
||||
}
|
||||
if (processedAsFolder(so, delimiter, prefix, folders)) {
|
||||
continue;
|
||||
}
|
||||
so.setSize(size(so));
|
||||
values.add(so);
|
||||
}
|
||||
values.addAll(folders.values());
|
||||
|
||||
// truncate to max allowed length
|
||||
if (values.size() > maxResults) {
|
||||
List<StorageObject> newValues = new ArrayList<>();
|
||||
for (int i = 0; i < maxResults; i++) {
|
||||
newValues.add(values.get(i));
|
||||
}
|
||||
values = newValues;
|
||||
}
|
||||
|
||||
// null cursor to indicate there is no more data (empty string would cause us to be called
|
||||
// again).
|
||||
// The type cast seems to be necessary to help Java's typesystem remember that collections are
|
||||
// iterable.
|
||||
return Tuple.of(pageToken, (Iterable<StorageObject>) values);
|
||||
}
|
||||
|
||||
/** Returns the requested bucket or {@code null} if not found. */
|
||||
@Override
|
||||
public Bucket get(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Returns the requested storage object or {@code null} if not found. */
|
||||
@Override
|
||||
public StorageObject get(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
// we allow the "ID" option because we need to, but then we give a whole answer anyways
|
||||
// because the caller won't mind the extra fields.
|
||||
if (throwIfOption
|
||||
&& !options.isEmpty()
|
||||
&& options.size() > 1
|
||||
&& options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
String key = fullname(object);
|
||||
if (metadata.containsKey(key)) {
|
||||
StorageObject ret = metadata.get(key);
|
||||
ret.setSize(size(ret));
|
||||
ret.setId(key);
|
||||
|
||||
return ret;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bucket patch(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject patch(StorageObject storageObject, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(object);
|
||||
contents.remove(key);
|
||||
return null != metadata.remove(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject compose(
|
||||
Iterable<StorageObject> sources, StorageObject target, Map<Option, ?> targetOptions)
|
||||
throws StorageException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] load(StorageObject storageObject, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(storageObject);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
return contents.get(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Tuple<String, byte[]> read(
|
||||
StorageObject from, Map<Option, ?> options, long zposition, int zbytes)
|
||||
throws StorageException {
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option op : options.keySet()) {
|
||||
if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) {
|
||||
generationMatch = (Long) options.get(op);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unknown option: " + op);
|
||||
}
|
||||
}
|
||||
String key = fullname(from);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
long position = zposition;
|
||||
int bytes = zbytes;
|
||||
if (position < 0) {
|
||||
position = 0;
|
||||
}
|
||||
byte[] full = contents.get(key);
|
||||
if ((int) position + bytes > full.length) {
|
||||
bytes = full.length - (int) position;
|
||||
}
|
||||
if (bytes <= 0) {
|
||||
// special case: you're trying to read past the end
|
||||
return Tuple.of("etag-goes-here", new byte[0]);
|
||||
}
|
||||
byte[] ret = new byte[bytes];
|
||||
System.arraycopy(full, (int) position, ret, 0, bytes);
|
||||
return Tuple.of("etag-goes-here", ret);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long read(
|
||||
StorageObject from, Map<Option, ?> options, long position, OutputStream outputStream) {
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option op : options.keySet()) {
|
||||
if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) {
|
||||
generationMatch = (Long) options.get(op);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unknown option: " + op);
|
||||
}
|
||||
}
|
||||
String key = fullname(from);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
if (position < 0) {
|
||||
position = 0;
|
||||
}
|
||||
byte[] full = contents.get(key);
|
||||
int bytes = (int) (full.length - position);
|
||||
if (bytes <= 0) {
|
||||
// special case: you're trying to read past the end
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
outputStream.write(full, (int) position, bytes);
|
||||
} catch (IOException e) {
|
||||
throw new StorageException(500, "Failed to write to file", e);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String open(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(object);
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option option : options.keySet()) {
|
||||
// this is a bit of a hack, since we don't implement generations.
|
||||
if (option == Option.IF_GENERATION_MATCH) {
|
||||
generationMatch = (Long) options.get(option);
|
||||
}
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
metadata.put(key, object);
|
||||
|
||||
return fullname(object);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String open(String signedURL) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(
|
||||
String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, int length, boolean last)
|
||||
throws StorageException {
|
||||
writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject writeWithResponse(
|
||||
String uploadId,
|
||||
byte[] toWrite,
|
||||
int toWriteOffset,
|
||||
long destOffset,
|
||||
int length,
|
||||
boolean last) {
|
||||
// this may have a lot more allocations than ideal, but it'll work.
|
||||
byte[] bytes;
|
||||
if (futureContents.containsKey(uploadId)) {
|
||||
bytes = futureContents.get(uploadId);
|
||||
if (bytes.length < length + destOffset) {
|
||||
byte[] newBytes = new byte[(int) (length + destOffset)];
|
||||
System.arraycopy(bytes, 0, newBytes, (int) 0, bytes.length);
|
||||
bytes = newBytes;
|
||||
}
|
||||
} else {
|
||||
bytes = new byte[(int) (length + destOffset)];
|
||||
}
|
||||
System.arraycopy(toWrite, toWriteOffset, bytes, (int) destOffset, length);
|
||||
// we want to mimic the GCS behavior that file contents are only visible on close.
|
||||
StorageObject storageObject = null;
|
||||
if (last) {
|
||||
contents.put(uploadId, bytes);
|
||||
futureContents.remove(uploadId);
|
||||
if (metadata.containsKey(uploadId)) {
|
||||
storageObject = metadata.get(uploadId);
|
||||
storageObject.setUpdated(now());
|
||||
Long generation = storageObject.getGeneration();
|
||||
if (null == generation) {
|
||||
generation = Long.valueOf(0);
|
||||
}
|
||||
storageObject.setGeneration(++generation);
|
||||
metadata.put(uploadId, storageObject);
|
||||
}
|
||||
} else {
|
||||
futureContents.put(uploadId, bytes);
|
||||
}
|
||||
return storageObject;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException {
|
||||
String sourceKey = fullname(rewriteRequest.source);
|
||||
|
||||
// a little hackish, just good enough for the tests to work.
|
||||
if (!contents.containsKey(sourceKey)) {
|
||||
throw new StorageException(404, "File not found: " + sourceKey);
|
||||
}
|
||||
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option option : rewriteRequest.targetOptions.keySet()) {
|
||||
// this is a bit of a hack, since we don't implement generations.
|
||||
if (option == Option.IF_GENERATION_MATCH) {
|
||||
generationMatch = (Long) rewriteRequest.targetOptions.get(option);
|
||||
}
|
||||
}
|
||||
|
||||
String destKey = fullname(rewriteRequest.target);
|
||||
|
||||
// if this is a new file, set generation to 1, else increment the existing generation
|
||||
long generation = 1;
|
||||
if (metadata.containsKey(destKey)) {
|
||||
generation = metadata.get(destKey).getGeneration() + 1;
|
||||
}
|
||||
|
||||
checkGeneration(destKey, generationMatch);
|
||||
|
||||
byte[] data = contents.get(sourceKey);
|
||||
|
||||
rewriteRequest.target.setGeneration(generation);
|
||||
rewriteRequest.target.setSize(BigInteger.valueOf(data.length));
|
||||
rewriteRequest.target.setUpdated(metadata.get(sourceKey).getUpdated());
|
||||
|
||||
metadata.put(destKey, rewriteRequest.target);
|
||||
|
||||
contents.put(destKey, Arrays.copyOf(data, data.length));
|
||||
return new RewriteResponse(
|
||||
rewriteRequest,
|
||||
rewriteRequest.target,
|
||||
data.length,
|
||||
true,
|
||||
"rewriteToken goes here",
|
||||
data.length);
|
||||
}
|
||||
|
||||
private static DateTime now() {
|
||||
return DateTime.parseRfc3339(RFC_3339_FORMATTER.format(new Date()));
|
||||
}
|
||||
|
||||
private String fullname(StorageObject so) {
|
||||
return (so.getBucket() + "/" + so.getName());
|
||||
}
|
||||
|
||||
private BigInteger size(StorageObject so) {
|
||||
String key = fullname(so);
|
||||
|
||||
if (contents.containsKey(key)) {
|
||||
return BigInteger.valueOf(contents.get(key).length);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private void potentiallyThrow(Map<Option, ?> options) throws UnsupportedOperationException {
|
||||
if (throwIfOption && !options.isEmpty()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw if we're asking for generation 0 and the file exists, or if the requested generation
|
||||
* number doesn't match what is asked.
|
||||
*
|
||||
* @param key
|
||||
* @param generationMatch
|
||||
*/
|
||||
private void checkGeneration(String key, Long generationMatch) {
|
||||
if (null == generationMatch) {
|
||||
return;
|
||||
}
|
||||
if (generationMatch == 0 && metadata.containsKey(key)) {
|
||||
throw new StorageException(new FileAlreadyExistsException(key));
|
||||
}
|
||||
if (generationMatch != 0) {
|
||||
Long generation = metadata.get(key).getGeneration();
|
||||
if (!generationMatch.equals(generation)) {
|
||||
throw new StorageException(
|
||||
404, "Generation mismatch. Requested " + generationMatch + " but got " + generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if this is a folder. Adds it to folders if it isn't already there.
|
||||
private static boolean processedAsFolder(
|
||||
StorageObject so,
|
||||
String delimiter,
|
||||
String prefix, /* inout */
|
||||
Map<String, StorageObject> folders) {
|
||||
if (delimiter == null) {
|
||||
return false;
|
||||
}
|
||||
int nextSlash = so.getName().indexOf(delimiter, prefix.length());
|
||||
if (nextSlash < 0) {
|
||||
return false;
|
||||
}
|
||||
String folderName = so.getName().substring(0, nextSlash + 1);
|
||||
if (folders.containsKey(folderName)) {
|
||||
return true;
|
||||
}
|
||||
StorageObject fakeFolder = new StorageObject();
|
||||
fakeFolder.setName(folderName);
|
||||
fakeFolder.setBucket(so.getBucket());
|
||||
fakeFolder.setGeneration(so.getGeneration());
|
||||
fakeFolder.set("isDirectory", true);
|
||||
fakeFolder.setSize(BigInteger.ZERO);
|
||||
folders.put(folderName, fakeFolder);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServiceAccount getServiceAccount(String projectId) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
* Copyright 2016 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package google.registry.gcs.backport;
|
||||
|
||||
import com.google.cloud.ServiceRpc;
|
||||
import com.google.cloud.spi.ServiceRpcFactory;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
|
||||
/**
|
||||
* Utility to create an in-memory storage configuration for testing. Storage options can be obtained
|
||||
* via the {@link #getOptions()} method. Returned options will point to FakeStorageRpc.
|
||||
*
|
||||
* <p>Note, the created in-memory storage configuration supports limited set of operations and is
|
||||
* <b>not</b> thread-safe:
|
||||
*
|
||||
* <ul>
|
||||
* <li>Supported operations
|
||||
* <ul>
|
||||
* <li>object create
|
||||
* <li>object get
|
||||
* <li>object delete
|
||||
* <li>list the contents of a bucket
|
||||
* </ul>
|
||||
* <li>Unsupported operations
|
||||
* <ul>
|
||||
* <li>bucket create
|
||||
* <li>bucket get
|
||||
* <li>bucket delete
|
||||
* <li>list all buckets
|
||||
* <li>generations
|
||||
* <li>file attributes
|
||||
* <li>patch
|
||||
* <li>continueRewrite
|
||||
* <li>createBatch
|
||||
* <li>checksums, etags
|
||||
* <li>IAM operations
|
||||
* </ul>
|
||||
* </ul>
|
||||
*
|
||||
* {@link FakeStorageRpc#list(String, java.util.Map)} lists all the objects that have been created
|
||||
* rather than the objects in the provided bucket. Since this class does not support creating,
|
||||
* listing and deleting buckets, the parameter bucket here is not actually used and on serves as a
|
||||
* placeholder.
|
||||
*/
|
||||
public final class LocalStorageHelper {
|
||||
|
||||
// used for testing. Will throw if you pass it an option.
|
||||
private static final FakeStorageRpc instance = new FakeStorageRpc(true);
|
||||
|
||||
private LocalStorageHelper() {}
|
||||
|
||||
/**
|
||||
* Returns a {@link StorageOptions} that use the static FakeStorageRpc instance, and resets it
|
||||
* first so you start from a clean slate. That instance will throw if you pass it any option.
|
||||
*/
|
||||
public static StorageOptions getOptions() {
|
||||
instance.reset();
|
||||
return StorageOptions.newBuilder()
|
||||
.setProjectId("fake-project-for-testing")
|
||||
.setServiceRpcFactory(new FakeStorageRpcFactory())
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link StorageOptions} that creates a new FakeStorageRpc instance with the given
|
||||
* option.
|
||||
*/
|
||||
public static StorageOptions customOptions(final boolean throwIfOptions) {
|
||||
return StorageOptions.newBuilder()
|
||||
.setProjectId("fake-project-for-testing")
|
||||
.setServiceRpcFactory(new FakeStorageRpcFactory(new FakeStorageRpc(throwIfOptions)))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static class FakeStorageRpcFactory implements ServiceRpcFactory<StorageOptions> {
|
||||
|
||||
private final FakeStorageRpc instance;
|
||||
|
||||
public FakeStorageRpcFactory() {
|
||||
this(LocalStorageHelper.instance);
|
||||
}
|
||||
|
||||
public FakeStorageRpcFactory(FakeStorageRpc instance) {
|
||||
this.instance = instance;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServiceRpc create(StorageOptions storageOptions) {
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
The files here are directly lifted
|
||||
from [googleapis/java-storage-nio](https://github.com/googleapis/java-storage-nio/tree/master/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing)
|
||||
. They are needed because the `StorageOptions` returned from `LocalStorageHelper.getOptions()`
|
||||
should be serializable for the `GcsUtils` class itself to be serializable in tests, but is not. The
|
||||
bug is [fixed](https://github.com/googleapis/java-storage-nio/pull/606) upstream. However, the
|
||||
current released package does not contain the fix yet.
|
||||
|
||||
They are not put under `common/testing` because we do not want to introduce a dependency on the
|
||||
testing configuration from core.
|
||||
+17
@@ -15,6 +15,7 @@
|
||||
package google.registry.model.common;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_ONLY;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_READ_ONLY;
|
||||
@@ -22,6 +23,7 @@ import static google.registry.model.common.DatabaseMigrationStateSchedule.Migrat
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY_READ_ONLY;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
|
||||
@@ -30,6 +32,7 @@ import google.registry.model.EntityTestCase;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -40,6 +43,11 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
|
||||
fakeClock.setAutoIncrementByOneMilli();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
ofyTm().transact(() -> DatabaseMigrationStateSchedule.set(DEFAULT_TRANSITION_MAP.toValueMap()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testEmpty_returnsDatastoreOnlyMap() {
|
||||
assertThat(DatabaseMigrationStateSchedule.getUncached())
|
||||
@@ -134,6 +142,15 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
|
||||
.isEqualTo("Must be called in a transaction");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_factoryUsesSchedule() {
|
||||
assertThat(tm().isOfy()).isTrue();
|
||||
// set the schedule to have converted to SQL_PRIMARY in the past
|
||||
fakeClock.setTo(START_OF_TIME.plusDays(1));
|
||||
runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY);
|
||||
assertThat(tm().isOfy()).isFalse();
|
||||
}
|
||||
|
||||
private void runValidTransition(MigrationState from, MigrationState to) {
|
||||
ImmutableSortedMap<DateTime, MigrationState> transitions =
|
||||
createMapEndingWithTransition(from, to);
|
||||
|
||||
+18
-10
@@ -17,7 +17,7 @@ package google.registry.persistence.transaction;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.testing.DatabaseHelper.assertDetached;
|
||||
import static google.registry.testing.DatabaseHelper.assertDetachedFromEntityManager;
|
||||
import static google.registry.testing.TestDataHelper.fileClassPath;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
@@ -388,7 +388,8 @@ class JpaTransactionManagerImplTest {
|
||||
void load_succeeds() {
|
||||
assertThat(jpaTm().transact(() -> jpaTm().exists(theEntity))).isFalse();
|
||||
jpaTm().transact(() -> jpaTm().insert(theEntity));
|
||||
TestEntity persisted = jpaTm().transact(() -> assertDetached(jpaTm().loadByKey(theEntityKey)));
|
||||
TestEntity persisted =
|
||||
jpaTm().transact(() -> assertDetachedFromEntityManager(jpaTm().loadByKey(theEntityKey)));
|
||||
assertThat(persisted.name).isEqualTo("theEntity");
|
||||
assertThat(persisted.data).isEqualTo("foo");
|
||||
}
|
||||
@@ -404,7 +405,8 @@ class JpaTransactionManagerImplTest {
|
||||
@Test
|
||||
void loadByEntity_succeeds() {
|
||||
jpaTm().transact(() -> jpaTm().insert(theEntity));
|
||||
TestEntity persisted = jpaTm().transact(() -> assertDetached(jpaTm().loadByEntity(theEntity)));
|
||||
TestEntity persisted =
|
||||
jpaTm().transact(() -> assertDetachedFromEntityManager(jpaTm().loadByEntity(theEntity)));
|
||||
assertThat(persisted.name).isEqualTo("theEntity");
|
||||
assertThat(persisted.data).isEqualTo("foo");
|
||||
}
|
||||
@@ -414,7 +416,11 @@ class JpaTransactionManagerImplTest {
|
||||
assertThat(jpaTm().transact(() -> jpaTm().exists(theEntity))).isFalse();
|
||||
jpaTm().transact(() -> jpaTm().insert(theEntity));
|
||||
TestEntity persisted =
|
||||
jpaTm().transact(() -> assertDetached(jpaTm().loadByKeyIfPresent(theEntityKey).get()));
|
||||
jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
assertDetachedFromEntityManager(
|
||||
jpaTm().loadByKeyIfPresent(theEntityKey).get()));
|
||||
assertThat(persisted.name).isEqualTo("theEntity");
|
||||
assertThat(persisted.data).isEqualTo("foo");
|
||||
}
|
||||
@@ -431,7 +437,9 @@ class JpaTransactionManagerImplTest {
|
||||
assertThat(jpaTm().transact(() -> jpaTm().exists(compoundIdEntity))).isFalse();
|
||||
jpaTm().transact(() -> jpaTm().insert(compoundIdEntity));
|
||||
TestCompoundIdEntity persisted =
|
||||
jpaTm().transact(() -> assertDetached(jpaTm().loadByKey(compoundIdEntityKey)));
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> assertDetachedFromEntityManager(jpaTm().loadByKey(compoundIdEntityKey)));
|
||||
assertThat(persisted.name).isEqualTo("compoundIdEntity");
|
||||
assertThat(persisted.age).isEqualTo(10);
|
||||
assertThat(persisted.data).isEqualTo("foo");
|
||||
@@ -450,7 +458,7 @@ class JpaTransactionManagerImplTest {
|
||||
theEntityKey, VKey.createSql(TestEntity.class, "does-not-exist")));
|
||||
|
||||
assertThat(results).containsExactly(theEntityKey, theEntity);
|
||||
assertDetached(results.get(theEntityKey));
|
||||
assertDetachedFromEntityManager(results.get(theEntityKey));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -463,7 +471,7 @@ class JpaTransactionManagerImplTest {
|
||||
ImmutableMap<VKey<? extends TestEntity>, TestEntity> results =
|
||||
jpaTm().loadByKeysIfPresent(ImmutableList.of(theEntityKey));
|
||||
assertThat(results).containsExactly(theEntityKey, theEntity);
|
||||
assertDetached(results.get(theEntityKey));
|
||||
assertDetachedFromEntityManager(results.get(theEntityKey));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -478,7 +486,7 @@ class JpaTransactionManagerImplTest {
|
||||
.loadByEntitiesIfPresent(
|
||||
ImmutableList.of(theEntity, new TestEntity("does-not-exist", "bar")));
|
||||
assertThat(results).containsExactly(theEntity);
|
||||
assertDetached(results.get(0));
|
||||
assertDetachedFromEntityManager(results.get(0));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -491,7 +499,7 @@ class JpaTransactionManagerImplTest {
|
||||
ImmutableList<TestEntity> results =
|
||||
jpaTm().loadByEntities(ImmutableList.of(theEntity));
|
||||
assertThat(results).containsExactly(theEntity);
|
||||
assertDetached(results.get(0));
|
||||
assertDetachedFromEntityManager(results.get(0));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -503,7 +511,7 @@ class JpaTransactionManagerImplTest {
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm().loadAllOf(TestEntity.class).stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(DatabaseHelper::assertDetachedFromEntityManager)
|
||||
.collect(toImmutableList()));
|
||||
assertThat(persisted).containsExactlyElementsIn(moreEntities);
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.GT, "bravo")
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(charlie);
|
||||
assertThat(
|
||||
@@ -86,7 +86,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.GTE, "charlie")
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(charlie);
|
||||
assertThat(
|
||||
@@ -95,7 +95,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.LT, "bravo")
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(alpha);
|
||||
assertThat(
|
||||
@@ -104,7 +104,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.LTE, "alpha")
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(alpha);
|
||||
}
|
||||
@@ -125,7 +125,7 @@ public class QueryComposerTest {
|
||||
assertThat(
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
DatabaseHelper.assertDetached(
|
||||
QueryComposerTest.assertDetachedIfJpa(
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.EQ, "alpha")
|
||||
.getSingleResult())))
|
||||
@@ -174,7 +174,7 @@ public class QueryComposerTest {
|
||||
.createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.GT, "alpha")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(bravo, charlie);
|
||||
assertThat(
|
||||
@@ -184,7 +184,7 @@ public class QueryComposerTest {
|
||||
.createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.GTE, "bravo")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(bravo, charlie);
|
||||
assertThat(
|
||||
@@ -194,7 +194,7 @@ public class QueryComposerTest {
|
||||
.createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.LT, "charlie")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(alpha, bravo);
|
||||
assertThat(
|
||||
@@ -204,7 +204,7 @@ public class QueryComposerTest {
|
||||
.createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.LTE, "bravo")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(alpha, bravo);
|
||||
}
|
||||
@@ -228,7 +228,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("val", Comparator.EQ, 2)
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(bravo);
|
||||
}
|
||||
@@ -243,7 +243,7 @@ public class QueryComposerTest {
|
||||
.where("val", Comparator.GT, 1)
|
||||
.orderBy("val")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(bravo, alpha);
|
||||
}
|
||||
@@ -354,6 +354,13 @@ public class QueryComposerTest {
|
||||
.contains("The LIKE operation is not supported on Datastore.");
|
||||
}
|
||||
|
||||
private static <T> T assertDetachedIfJpa(T entity) {
|
||||
if (!tm().isOfy()) {
|
||||
return DatabaseHelper.assertDetachedFromEntityManager(entity);
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
@javax.persistence.Entity
|
||||
@Entity(name = "QueryComposerTestEntity")
|
||||
private static class TestEntity extends ImmutableObject {
|
||||
|
||||
@@ -16,23 +16,20 @@ package google.registry.rde;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth.assertWithMessage;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.SystemInfo.hasCommand;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assumptions.assumeTrue;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.io.ByteSource;
|
||||
import com.google.common.io.CharStreams;
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.BouncyCastleProviderExtension;
|
||||
import google.registry.testing.FakeKeyringModule;
|
||||
import google.registry.testing.GcsTestingUtils;
|
||||
import google.registry.testing.GpgSystemCommandExtension;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
@@ -52,14 +49,12 @@ public class BrdaCopyActionTest {
|
||||
|
||||
private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml");
|
||||
|
||||
private static final GcsFilename STAGE_FILE =
|
||||
new GcsFilename("keg", "lol_2010-10-17_thin_S1_R0.xml.ghostryde");
|
||||
private static final GcsFilename STAGE_LENGTH_FILE =
|
||||
new GcsFilename("keg", "lol_2010-10-17_thin_S1_R0.xml.length");
|
||||
private static final GcsFilename RYDE_FILE =
|
||||
new GcsFilename("tub", "lol_2010-10-17_thin_S1_R0.ryde");
|
||||
private static final GcsFilename SIG_FILE =
|
||||
new GcsFilename("tub", "lol_2010-10-17_thin_S1_R0.sig");
|
||||
private static final BlobId STAGE_FILE =
|
||||
BlobId.of("keg", "lol_2010-10-17_thin_S1_R0.xml.ghostryde");
|
||||
private static final BlobId STAGE_LENGTH_FILE =
|
||||
BlobId.of("keg", "lol_2010-10-17_thin_S1_R0.xml.length");
|
||||
private static final BlobId RYDE_FILE = BlobId.of("tub", "lol_2010-10-17_thin_S1_R0.ryde");
|
||||
private static final BlobId SIG_FILE = BlobId.of("tub", "lol_2010-10-17_thin_S1_R0.sig");
|
||||
|
||||
@RegisterExtension
|
||||
public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension();
|
||||
@@ -89,8 +84,7 @@ public class BrdaCopyActionTest {
|
||||
}
|
||||
}
|
||||
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final BrdaCopyAction action = new BrdaCopyAction();
|
||||
|
||||
@BeforeEach
|
||||
@@ -105,9 +99,8 @@ public class BrdaCopyActionTest {
|
||||
action.stagingDecryptionKey = decryptKey;
|
||||
|
||||
byte[] xml = DEPOSIT_XML.read();
|
||||
GcsTestingUtils.writeGcsFile(gcsService, STAGE_FILE, Ghostryde.encode(xml, encryptKey));
|
||||
GcsTestingUtils.writeGcsFile(gcsService, STAGE_LENGTH_FILE,
|
||||
Long.toString(xml.length).getBytes(UTF_8));
|
||||
gcsUtils.createFromBytes(STAGE_FILE, Ghostryde.encode(xml, encryptKey));
|
||||
gcsUtils.createFromBytes(STAGE_LENGTH_FILE, Long.toString(xml.length).getBytes(UTF_8));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -124,7 +117,7 @@ public class BrdaCopyActionTest {
|
||||
action.run();
|
||||
|
||||
File rydeTmp = new File(gpg.getCwd(), "ryde");
|
||||
Files.write(readGcsFile(gcsService, RYDE_FILE), rydeTmp);
|
||||
Files.write(gcsUtils.readBytesFrom(RYDE_FILE), rydeTmp);
|
||||
Process pid =
|
||||
gpg.exec(
|
||||
"gpg",
|
||||
@@ -172,8 +165,8 @@ public class BrdaCopyActionTest {
|
||||
|
||||
File rydeTmp = new File(gpg.getCwd(), "ryde");
|
||||
File sigTmp = new File(gpg.getCwd(), "ryde.sig");
|
||||
Files.write(readGcsFile(gcsService, RYDE_FILE), rydeTmp);
|
||||
Files.write(readGcsFile(gcsService, SIG_FILE), sigTmp);
|
||||
Files.write(gcsUtils.readBytesFrom(RYDE_FILE), rydeTmp);
|
||||
Files.write(gcsUtils.readBytesFrom(SIG_FILE), sigTmp);
|
||||
|
||||
Process pid = gpg.exec("gpg", "--verify", sigTmp.toString(), rydeTmp.toString());
|
||||
String stderr = slurp(pid.getErrorStream());
|
||||
|
||||
@@ -24,8 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.loadByKey;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.deleteGcsFile;
|
||||
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.joda.time.Duration.standardDays;
|
||||
@@ -41,13 +39,12 @@ import com.google.appengine.api.urlfetch.HTTPHeader;
|
||||
import com.google.appengine.api.urlfetch.HTTPRequest;
|
||||
import com.google.appengine.api.urlfetch.HTTPResponse;
|
||||
import com.google.appengine.api.urlfetch.URLFetchService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteSource;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.rde.RdeRevision;
|
||||
import google.registry.model.registry.Registry;
|
||||
@@ -95,9 +92,9 @@ public class RdeReportActionTest {
|
||||
private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
|
||||
private final HTTPResponse httpResponse = mock(HTTPResponse.class);
|
||||
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsFilename reportFile =
|
||||
new GcsFilename("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final BlobId reportFile =
|
||||
BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
|
||||
|
||||
private RdeReportAction createAction() {
|
||||
RdeReporter reporter = new RdeReporter();
|
||||
@@ -106,7 +103,7 @@ public class RdeReportActionTest {
|
||||
reporter.password = "foo";
|
||||
reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
|
||||
RdeReportAction action = new RdeReportAction();
|
||||
action.gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.response = response;
|
||||
action.bucket = "tub";
|
||||
action.tld = "test";
|
||||
@@ -126,7 +123,7 @@ public class RdeReportActionTest {
|
||||
Cursor.create(RDE_REPORT, DateTime.parse("2006-06-06TZ"), Registry.get("test")));
|
||||
persistResource(
|
||||
Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
|
||||
writeGcsFile(gcsService, reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
|
||||
}
|
||||
|
||||
@@ -169,11 +166,10 @@ public class RdeReportActionTest {
|
||||
|
||||
@TestOfyAndSql
|
||||
void testRunWithLock_regeneratedReport() throws Exception {
|
||||
deleteGcsFile(gcsService, reportFile);
|
||||
GcsFilename newReport =
|
||||
new GcsFilename("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde");
|
||||
gcsUtils.delete(reportFile);
|
||||
BlobId newReport = BlobId.of("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde");
|
||||
PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
|
||||
writeGcsFile(gcsService, newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
|
||||
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
|
||||
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
package google.registry.rde;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.common.Cursor.CursorType.BRDA;
|
||||
import static google.registry.model.common.Cursor.CursorType.RDE_STAGING;
|
||||
@@ -26,7 +25,6 @@ import static google.registry.rde.RdeFixtures.makeHostResource;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.DatabaseHelper.persistResourceWithCommitLog;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.TaskQueueHelper.assertAtLeastOneTaskIsEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
@@ -36,16 +34,13 @@ import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static java.util.Arrays.asList;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.net.InetAddresses;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
@@ -98,16 +93,16 @@ import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
/** Unit tests for {@link RdeStagingAction}. */
|
||||
public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
|
||||
private static final GcsFilename XML_FILE =
|
||||
new GcsFilename("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
private static final GcsFilename LENGTH_FILE =
|
||||
new GcsFilename("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.length");
|
||||
private static final BlobId XML_FILE =
|
||||
BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
private static final BlobId LENGTH_FILE =
|
||||
BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.length");
|
||||
|
||||
@RegisterExtension public final InjectExtension inject = new InjectExtension();
|
||||
|
||||
private final FakeClock clock = new FakeClock();
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final List<? super XjcRdeContentType> alreadyExtracted = new ArrayList<>();
|
||||
|
||||
private static PGPPublicKey encryptKey;
|
||||
@@ -131,7 +126,6 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
action.reducerFactory = new RdeStagingReducer.Factory();
|
||||
action.reducerFactory.taskQueueUtils = new TaskQueueUtils(new Retrier(new SystemSleeper(), 1));
|
||||
action.reducerFactory.lockHandler = new FakeLockHandler(true);
|
||||
action.reducerFactory.gcsBufferSize = 0;
|
||||
action.reducerFactory.bucket = "rde-bucket";
|
||||
action.reducerFactory.lockTimeout = Duration.standardHours(1);
|
||||
action.reducerFactory.stagingKeyBytes = PgpHelper.convertPublicKeyToBytes(encryptKey);
|
||||
@@ -140,6 +134,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
action.pendingDepositChecker.brdaInterval = Duration.standardDays(7);
|
||||
action.pendingDepositChecker.clock = clock;
|
||||
action.pendingDepositChecker.rdeInterval = Duration.standardDays(1);
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.response = response;
|
||||
action.transactionCooldown = Duration.ZERO;
|
||||
action.directory = Optional.empty();
|
||||
@@ -327,7 +322,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey));
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
|
||||
assertThat(header.getTld()).isEqualTo("lol");
|
||||
@@ -358,7 +353,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey));
|
||||
assertThat(deposit.getType()).isEqualTo(XjcRdeDepositTypeType.FULL);
|
||||
assertThat(deposit.getId()).isEqualTo(RdeUtil.timestampToId(DateTime.parse("2000-01-01TZ")));
|
||||
assertThat(deposit.getWatermark()).isEqualTo(DateTime.parse("2000-01-01TZ"));
|
||||
@@ -400,7 +395,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey));
|
||||
XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
@@ -494,12 +489,13 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
for (GcsFilename filename : asList(
|
||||
new GcsFilename("rde-bucket", "fop_1971-01-01_full_S1_R0.xml.ghostryde"),
|
||||
new GcsFilename("rde-bucket", "fop_1971-01-05_thin_S1_R0.xml.ghostryde"))) {
|
||||
for (BlobId filename :
|
||||
asList(
|
||||
BlobId.of("rde-bucket", "fop_1971-01-01_full_S1_R0.xml.ghostryde"),
|
||||
BlobId.of("rde-bucket", "fop_1971-01-05_thin_S1_R0.xml.ghostryde"))) {
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(filename), decryptKey));
|
||||
XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
@@ -536,10 +532,10 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
GcsFilename filename = new GcsFilename("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
BlobId filename = BlobId.of("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(filename), decryptKey));
|
||||
XjcRdeDomain domain = extractAndRemoveContentWithType(XjcRdeDomain.class, deposit);
|
||||
XjcRdeIdn firstIdn = extractAndRemoveContentWithType(XjcRdeIdn.class, deposit);
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
@@ -579,8 +575,8 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
byte[] deposit = Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey);
|
||||
assertThat(Integer.parseInt(new String(readGcsFile(gcsService, LENGTH_FILE), UTF_8)))
|
||||
byte[] deposit = Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey);
|
||||
assertThat(Integer.parseInt(new String(gcsUtils.readBytesFrom(LENGTH_FILE), UTF_8)))
|
||||
.isEqualTo(deposit.length);
|
||||
}
|
||||
|
||||
@@ -781,23 +777,20 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
ListResult listResult =
|
||||
gcsService.list("rde-bucket", new ListOptions.Builder().setPrefix("manual/test").build());
|
||||
ImmutableSet<String> filenames =
|
||||
ImmutableList.copyOf(listResult).stream().map(ListItem::getName).collect(toImmutableSet());
|
||||
ImmutableList<String> filenames = gcsUtils.listFolderObjects("rde-bucket", "manual/test/");
|
||||
for (String tld : tlds) {
|
||||
assertThat(filenames)
|
||||
.containsAtLeast(
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length");
|
||||
tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde",
|
||||
tld + "_2000-01-01_full_S1_R" + revision + ".xml.length",
|
||||
tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length",
|
||||
tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde",
|
||||
tld + "_2000-01-02_full_S1_R" + revision + ".xml.length",
|
||||
tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length");
|
||||
|
||||
assertThat(
|
||||
auditedOfy()
|
||||
@@ -837,8 +830,8 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
}
|
||||
|
||||
private String readXml(String objectName) throws IOException, PGPException {
|
||||
GcsFilename file = new GcsFilename("rde-bucket", objectName);
|
||||
return new String(Ghostryde.decode(readGcsFile(gcsService, file), decryptKey), UTF_8);
|
||||
BlobId file = BlobId.of("rde-bucket", objectName);
|
||||
return new String(Ghostryde.decode(gcsUtils.readBytesFrom(file), decryptKey), UTF_8);
|
||||
}
|
||||
|
||||
private <T extends XjcRdeContentType>
|
||||
|
||||
@@ -20,18 +20,18 @@ import static google.registry.model.rde.RdeMode.FULL;
|
||||
import static google.registry.model.rde.RdeMode.THIN;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static google.registry.util.ResourceUtils.readResourceUtf8;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
@@ -67,7 +67,7 @@ class RdeStagingReducerTest {
|
||||
AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
|
||||
|
||||
private static final String GCS_BUCKET = "test-rde-bucket";
|
||||
private static final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private static final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private static final PGPPrivateKey decryptionKey =
|
||||
new FakeKeyringModule().get().getRdeStagingDecryptionKey();
|
||||
private static final PGPPublicKey encryptionKey =
|
||||
@@ -95,11 +95,11 @@ class RdeStagingReducerTest {
|
||||
new RdeStagingReducer(
|
||||
new TaskQueueUtils(new Retrier(new FakeSleeper(new FakeClock()), 1)),
|
||||
new FakeLockHandler(true),
|
||||
1024,
|
||||
GCS_BUCKET,
|
||||
Duration.ZERO,
|
||||
PgpHelper.convertPublicKeyToBytes(encryptionKey),
|
||||
ValidationMode.STRICT);
|
||||
ValidationMode.STRICT,
|
||||
gcsUtils);
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
@@ -125,11 +125,10 @@ class RdeStagingReducerTest {
|
||||
compareLength(outputFile, "soy_2000-01-01_thin_S1_R1.xml.length");
|
||||
// BRDA doesn't write a report file.
|
||||
assertThrows(
|
||||
IOException.class,
|
||||
StorageException.class,
|
||||
() ->
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename(GCS_BUCKET, "soy_2000-01-01_thin_S1_R1-report.xml.ghostryde")));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of(GCS_BUCKET, "soy_2000-01-01_thin_S1_R1-report.xml.ghostryde")));
|
||||
assertThat(loadCursorTime(CursorType.BRDA))
|
||||
.isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
|
||||
assertThat(loadRevision(THIN)).isEqualTo(1);
|
||||
@@ -153,12 +152,10 @@ class RdeStagingReducerTest {
|
||||
compareLength(outputFile, "manual/soy_2000-01-01_thin_S1_R0.xml.length");
|
||||
// BRDA doesn't write a report file.
|
||||
assertThrows(
|
||||
IOException.class,
|
||||
StorageException.class,
|
||||
() ->
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename(
|
||||
GCS_BUCKET, "manual/soy_2000-01-01_thin_S1_R0-report.xml.ghostryde")));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of(GCS_BUCKET, "manual/soy_2000-01-01_thin_S1_R0-report.xml.ghostryde")));
|
||||
// No extra operations in manual mode.
|
||||
assertThat(loadCursorTime(CursorType.BRDA)).isEquivalentAccordingToCompareTo(now);
|
||||
assertThat(loadRevision(THIN)).isEqualTo(0);
|
||||
@@ -210,8 +207,7 @@ class RdeStagingReducerTest {
|
||||
private static void compareLength(String outputFile, String lengthFilename) throws IOException {
|
||||
assertThat(String.valueOf(outputFile.getBytes(UTF_8).length))
|
||||
.isEqualTo(
|
||||
new String(
|
||||
readGcsFile(gcsService, new GcsFilename(GCS_BUCKET, lengthFilename)), UTF_8));
|
||||
new String(gcsUtils.readBytesFrom(BlobId.of(GCS_BUCKET, lengthFilename)), UTF_8));
|
||||
}
|
||||
|
||||
private static DateTime loadCursorTime(CursorType type) {
|
||||
@@ -233,8 +229,7 @@ class RdeStagingReducerTest {
|
||||
|
||||
private static String decryptGhostrydeGcsFile(String filename) throws IOException, PGPException {
|
||||
return new String(
|
||||
Ghostryde.decode(
|
||||
readGcsFile(gcsService, new GcsFilename(GCS_BUCKET, filename)), decryptionKey),
|
||||
Ghostryde.decode(gcsUtils.readBytesFrom(BlobId.of(GCS_BUCKET, filename)), decryptionKey),
|
||||
UTF_8);
|
||||
}
|
||||
|
||||
|
||||
@@ -24,8 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.DatabaseHelper.persistSimpleResource;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
|
||||
import static google.registry.testing.SystemInfo.hasCommand;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
@@ -45,9 +43,7 @@ import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.utils.SystemProperty;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.io.ByteSource;
|
||||
import com.google.common.io.CharStreams;
|
||||
import com.google.common.io.Files;
|
||||
@@ -55,6 +51,7 @@ import com.jcraft.jsch.JSch;
|
||||
import com.jcraft.jsch.JSchException;
|
||||
import com.jcraft.jsch.Session;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
@@ -94,23 +91,24 @@ import org.mockito.stubbing.OngoingStubbing;
|
||||
@DualDatabaseTest
|
||||
public class RdeUploadActionTest {
|
||||
|
||||
private static final int BUFFER_SIZE = 64 * 1024;
|
||||
private static final ByteSource REPORT_XML = RdeTestData.loadBytes("report.xml");
|
||||
private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml");
|
||||
|
||||
private static final GcsFilename GHOSTRYDE_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde");
|
||||
private static final GcsFilename LENGTH_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0.xml.length");
|
||||
private static final GcsFilename REPORT_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde");
|
||||
private static final BlobId GHOSTRYDE_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde");
|
||||
private static final BlobId LENGTH_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.length");
|
||||
private static final BlobId REPORT_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde");
|
||||
|
||||
private static final GcsFilename GHOSTRYDE_R1_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde");
|
||||
private static final GcsFilename LENGTH_R1_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1.xml.length");
|
||||
private static final GcsFilename REPORT_R1_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1-report.xml.ghostryde");
|
||||
private static final BlobId GHOSTRYDE_R1_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde");
|
||||
private static final BlobId LENGTH_R1_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.length");
|
||||
private static final BlobId REPORT_R1_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R1-report.xml.ghostryde");
|
||||
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
@RegisterExtension final SftpServerExtension sftpd = new SftpServerExtension();
|
||||
|
||||
@@ -139,7 +137,7 @@ public class RdeUploadActionTest {
|
||||
try (Keyring keyring = new FakeKeyringModule().get()) {
|
||||
RdeUploadAction action = new RdeUploadAction();
|
||||
action.clock = clock;
|
||||
action.gcsUtils = new GcsUtils(gcsService, BUFFER_SIZE);
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.lazyJsch =
|
||||
() ->
|
||||
JSchModule.provideJSch(
|
||||
@@ -176,22 +174,20 @@ public class RdeUploadActionTest {
|
||||
return jschSpy;
|
||||
}
|
||||
|
||||
private GcsService gcsService;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
// Force "development" mode so we don't try to really connect to GCS.
|
||||
SystemProperty.environment.set(SystemProperty.Environment.Value.Development);
|
||||
gcsService = GcsServiceFactory.createGcsService();
|
||||
|
||||
createTld("tld");
|
||||
PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
|
||||
writeGcsFile(gcsService, GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
|
||||
writeGcsFile(gcsService, GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
|
||||
writeGcsFile(gcsService, LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
|
||||
writeGcsFile(gcsService, LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
|
||||
writeGcsFile(gcsService, REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
writeGcsFile(gcsService, REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
|
||||
gcsUtils.createFromBytes(LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
|
||||
gcsUtils.createFromBytes(REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
tm()
|
||||
.transact(
|
||||
() -> {
|
||||
@@ -271,9 +267,9 @@ public class RdeUploadActionTest {
|
||||
String rydeFilename = "tld_2010-10-17_full_S1_R0.ryde";
|
||||
String sigFilename = "tld_2010-10-17_full_S1_R0.sig";
|
||||
assertThat(folder.list()).asList().containsExactly(rydeFilename, sigFilename);
|
||||
assertThat(readGcsFile(gcsService, new GcsFilename("bucket", rydeFilename)))
|
||||
assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", rydeFilename)))
|
||||
.isEqualTo(Files.toByteArray(new File(folder, rydeFilename)));
|
||||
assertThat(readGcsFile(gcsService, new GcsFilename("bucket", sigFilename)))
|
||||
assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", sigFilename)))
|
||||
.isEqualTo(Files.toByteArray(new File(folder, sigFilename)));
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
@@ -51,7 +51,7 @@ class BillingEmailUtilsTest {
|
||||
void beforeEach() throws Exception {
|
||||
emailService = mock(SendEmailService.class);
|
||||
gcsUtils = mock(GcsUtils.class);
|
||||
when(gcsUtils.openInputStream(new GcsFilename("test-bucket", "results/REG-INV-2017-10.csv")))
|
||||
when(gcsUtils.openInputStream(BlobId.of("test-bucket", "results/REG-INV-2017-10.csv")))
|
||||
.thenReturn(
|
||||
new ByteArrayInputStream("test,data\nhello,world".getBytes(StandardCharsets.UTF_8)));
|
||||
contentCaptor = ArgumentCaptor.forClass(EmailMessage.class);
|
||||
|
||||
+22
-33
@@ -17,7 +17,6 @@ package google.registry.reporting.billing;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.DatabaseHelper.loadRegistrar;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
@@ -29,11 +28,10 @@ import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
@@ -52,8 +50,7 @@ class CopyDetailReportsActionTest {
|
||||
final AppEngineExtension appEngine =
|
||||
AppEngineExtension.builder().withDatastoreAndCloudSql().build();
|
||||
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private FakeResponse response;
|
||||
private DriveConnection driveConnection;
|
||||
@@ -80,14 +77,12 @@ class CopyDetailReportsActionTest {
|
||||
|
||||
@Test
|
||||
void testSuccess() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"),
|
||||
"hello,world\n1,2".getBytes(UTF_8));
|
||||
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
|
||||
action.run();
|
||||
@@ -111,14 +106,12 @@ class CopyDetailReportsActionTest {
|
||||
|
||||
@Test
|
||||
void testSuccess_nonDetailReportFiles_notSent() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"),
|
||||
"hello,world\n1,2".getBytes(UTF_8));
|
||||
action.run();
|
||||
verify(driveConnection)
|
||||
@@ -136,9 +129,8 @@ class CopyDetailReportsActionTest {
|
||||
|
||||
@Test
|
||||
void testSuccess_transientIOException_retries() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
when(driveConnection.createOrUpdateFile(any(), any(), any(), any()))
|
||||
.thenThrow(new IOException("expected"))
|
||||
@@ -158,13 +150,12 @@ class CopyDetailReportsActionTest {
|
||||
|
||||
@Test
|
||||
void testFail_tooManyFailures_sendsAlertEmail_continues() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_NewRegistrar_test.csv"),
|
||||
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_NewRegistrar_test.csv"),
|
||||
"hello,world\n1,2".getBytes(UTF_8));
|
||||
when(driveConnection.createOrUpdateFile(
|
||||
eq("invoice_details_2017-10_TheRegistrar_hello.csv"), any(), any(), any()))
|
||||
@@ -201,9 +192,8 @@ class CopyDetailReportsActionTest {
|
||||
|
||||
@Test
|
||||
void testFail_registrarDoesntExist_doesntCopy() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
action.run();
|
||||
verifyNoInteractions(driveConnection);
|
||||
@@ -212,9 +202,8 @@ class CopyDetailReportsActionTest {
|
||||
@Test
|
||||
void testFail_noRegistrarFolderId_doesntCopy() throws IOException {
|
||||
persistResource(loadRegistrar("TheRegistrar").asBuilder().setDriveFolderId(null).build());
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
action.run();
|
||||
verifyNoInteractions(driveConnection);
|
||||
|
||||
@@ -15,16 +15,13 @@
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableTable;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
@@ -32,6 +29,7 @@ import google.registry.bigquery.BigqueryConnection;
|
||||
import google.registry.bigquery.BigqueryConnection.DestinationTable;
|
||||
import google.registry.bigquery.BigqueryUtils.TableType;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportType;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.FakeResponse;
|
||||
@@ -46,9 +44,9 @@ class IcannReportingStagerTest {
|
||||
|
||||
private BigqueryConnection bigquery = mock(BigqueryConnection.class);
|
||||
FakeResponse response = new FakeResponse();
|
||||
private GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private YearMonth yearMonth = new YearMonth(2017, 6);
|
||||
private String subdir = "icann/monthly/2017-06";
|
||||
private GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
@RegisterExtension
|
||||
final AppEngineExtension appEngine =
|
||||
@@ -65,7 +63,7 @@ class IcannReportingStagerTest {
|
||||
action.transactionsQueryBuilder = transactionsBuilder;
|
||||
action.reportingBucket = "test-bucket";
|
||||
action.bigquery = bigquery;
|
||||
action.gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
action.gcsUtils = gcsUtils;
|
||||
return action;
|
||||
}
|
||||
|
||||
@@ -100,14 +98,12 @@ class IcannReportingStagerTest {
|
||||
String expectedReport1 = "fooField,barField\r\n12,34";
|
||||
String expectedReport2 = "fooField,barField\r\n56,78";
|
||||
byte[] generatedFile1 =
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
|
||||
assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
|
||||
byte[] generatedFile2 =
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
|
||||
assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
|
||||
}
|
||||
|
||||
@@ -144,14 +140,12 @@ class IcannReportingStagerTest {
|
||||
"registrar,iana,field\r\n\"reg1\",123,10\r\n\"reg2\",456,20\r\nTotals,,30";
|
||||
String expectedReport2 = "registrar,iana,field\r\n\"reg1\",123,30\r\nTotals,,30";
|
||||
byte[] generatedFile1 =
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
|
||||
assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
|
||||
byte[] generatedFile2 =
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
|
||||
assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
|
||||
}
|
||||
|
||||
@@ -164,8 +158,7 @@ class IcannReportingStagerTest {
|
||||
|
||||
String expectedManifest = "fooTld-transactions-201706.csv\nbarTld-activity-201706.csv\n";
|
||||
byte[] generatedManifest =
|
||||
readGcsFile(
|
||||
gcsService, new GcsFilename("test-bucket/icann/monthly/2017-06", "MANIFEST.txt"));
|
||||
gcsUtils.readBytesFrom(BlobId.of("test-bucket/icann/monthly/2017-06", "MANIFEST.txt"));
|
||||
assertThat(new String(generatedManifest, UTF_8)).isEqualTo(expectedManifest);
|
||||
}
|
||||
|
||||
|
||||
+18
-34
@@ -19,7 +19,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.testing.DatabaseHelper.createTlds;
|
||||
import static google.registry.testing.DatabaseHelper.loadByKey;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
|
||||
import static google.registry.testing.LogsSubject.assertAboutLogs;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
@@ -29,11 +28,10 @@ import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.testing.TestLogHandler;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
import google.registry.model.registry.Registry;
|
||||
@@ -69,7 +67,7 @@ class IcannReportingUploadActionTest {
|
||||
private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class);
|
||||
private final SendEmailService emailService = mock(SendEmailService.class);
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final TestLogHandler logHandler = new TestLogHandler();
|
||||
private final Logger loggerToIntercept =
|
||||
Logger.getLogger(IcannReportingUploadAction.class.getCanonicalName());
|
||||
@@ -78,7 +76,7 @@ class IcannReportingUploadActionTest {
|
||||
private IcannReportingUploadAction createAction() throws Exception {
|
||||
IcannReportingUploadAction action = new IcannReportingUploadAction();
|
||||
action.icannReporter = mockReporter;
|
||||
action.gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
|
||||
action.reportingBucket = "basin";
|
||||
action.emailService = emailService;
|
||||
@@ -93,22 +91,14 @@ class IcannReportingUploadActionTest {
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
createTlds("tld", "foo");
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"),
|
||||
PAYLOAD_FAIL);
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "foo-activity-200606.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"), PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_FAIL);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"), PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "foo-activity-200606.csv"), PAYLOAD_SUCCESS);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv")).thenReturn(true);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv")).thenReturn(true);
|
||||
when(mockReporter.send(PAYLOAD_FAIL, "tld-activity-200606.csv")).thenReturn(false);
|
||||
@@ -161,14 +151,10 @@ class IcannReportingUploadActionTest {
|
||||
persistResource(
|
||||
Cursor.create(
|
||||
CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-01-01TZ"), Registry.get("tld")));
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2005-12", "tld-activity-200512.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"), PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2005-12", "tld-activity-200512.csv"), PAYLOAD_SUCCESS);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200512.csv")).thenReturn(true);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200512.csv")).thenReturn(true);
|
||||
|
||||
@@ -191,10 +177,8 @@ class IcannReportingUploadActionTest {
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_advancesCursor() throws Exception {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_SUCCESS);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200606.csv")).thenReturn(true);
|
||||
IcannReportingUploadAction action = createAction();
|
||||
action.run();
|
||||
|
||||
+3
-3
@@ -19,7 +19,7 @@ import static com.google.common.truth.Truth8.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
@@ -133,8 +133,8 @@ public class Spec11RegistrarThreatMatchesParserTest {
|
||||
}
|
||||
|
||||
private void setupFile(String fileWithContent, String fileDate) {
|
||||
GcsFilename gcsFilename =
|
||||
new GcsFilename(
|
||||
BlobId gcsFilename =
|
||||
BlobId.of(
|
||||
"test-bucket",
|
||||
String.format("icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_%s", fileDate));
|
||||
when(gcsUtils.existsAndNotEmpty(gcsFilename)).thenReturn(true);
|
||||
|
||||
@@ -96,6 +96,13 @@ public class ReplicateToDatastoreActionTest {
|
||||
public void tearDown() {
|
||||
fakeClock.disableAutoIncrement();
|
||||
RegistryConfig.overrideCloudSqlReplicateTransactions(false);
|
||||
ofyTm()
|
||||
.transact(
|
||||
() ->
|
||||
ofyTm()
|
||||
.loadSingleton(DatabaseMigrationStateSchedule.class)
|
||||
.ifPresent(ofyTm()::delete));
|
||||
DatabaseMigrationStateSchedule.CACHE.invalidateAll();
|
||||
}
|
||||
|
||||
@RetryingTest(4)
|
||||
|
||||
@@ -1351,14 +1351,12 @@ public class DatabaseHelper {
|
||||
}
|
||||
|
||||
/**
|
||||
* In JPA mode, assert that the given entity is detached from the current entity manager.
|
||||
* Asserts that the given entity is detached from the current JPA entity manager.
|
||||
*
|
||||
* <p>Returns the original entity object.
|
||||
*/
|
||||
public static <T> T assertDetached(T entity) {
|
||||
if (!tm().isOfy()) {
|
||||
assertThat(jpaTm().getEntityManager().contains(entity)).isFalse();
|
||||
}
|
||||
public static <T> T assertDetachedFromEntityManager(T entity) {
|
||||
assertThat(jpaTm().getEntityManager().contains(entity)).isFalse();
|
||||
return entity;
|
||||
}
|
||||
|
||||
|
||||
+2
-4
@@ -154,15 +154,13 @@ class DualDatabaseTestInvocationContextProvider implements TestTemplateInvocatio
|
||||
context.getStore(NAMESPACE).put(ORIGINAL_TM_KEY, tm());
|
||||
DatabaseType databaseType =
|
||||
(DatabaseType) context.getStore(NAMESPACE).get(INJECTED_TM_SUPPLIER_KEY);
|
||||
TransactionManagerFactory.setTm(databaseType.getTm());
|
||||
TransactionManagerFactory.setTmForTest(databaseType.getTm());
|
||||
}
|
||||
}
|
||||
|
||||
static void restoreTmAfterDualDatabaseTest(ExtensionContext context) {
|
||||
if (isDualDatabaseTest(context)) {
|
||||
TransactionManager original =
|
||||
(TransactionManager) context.getStore(NAMESPACE).get(ORIGINAL_TM_KEY);
|
||||
TransactionManagerFactory.setTm(original);
|
||||
TransactionManagerFactory.removeTmOverrideForTest();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.testing;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.Channels;
|
||||
|
||||
/** Utility methods for testing Google Cloud Storage code. */
|
||||
public final class GcsTestingUtils {
|
||||
|
||||
/** Slurps a Cloud Storage file into memory. */
|
||||
public static byte[] readGcsFile(GcsService gcsService, GcsFilename file)
|
||||
throws IOException {
|
||||
try (InputStream input = Channels.newInputStream(gcsService.openReadChannel(file, 0))) {
|
||||
return ByteStreams.toByteArray(input);
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes a Cloud Storage file. */
|
||||
public static void writeGcsFile(GcsService gcsService, GcsFilename file, byte[] data)
|
||||
throws IOException {
|
||||
gcsService.createOrReplace(file, GcsFileOptions.getDefaultInstance(), ByteBuffer.wrap(data));
|
||||
}
|
||||
|
||||
public static void deleteGcsFile(GcsService gcsService, GcsFilename file) throws IOException {
|
||||
gcsService.delete(file);
|
||||
}
|
||||
|
||||
private GcsTestingUtils() {}
|
||||
}
|
||||
@@ -24,15 +24,15 @@ import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationStat
|
||||
import google.registry.testing.DualDatabaseTest;
|
||||
import google.registry.testing.TestOfyAndSql;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
|
||||
/** Tests for {@link GetDatabaseMigrationStateCommand}. */
|
||||
@DualDatabaseTest
|
||||
public class GetDatabaseMigrationStateCommandTest
|
||||
extends CommandTestCase<GetDatabaseMigrationStateCommand> {
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
ofyTm().transact(() -> DatabaseMigrationStateSchedule.set(DEFAULT_TRANSITION_MAP.toValueMap()));
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationStat
|
||||
import google.registry.testing.DualDatabaseTest;
|
||||
import google.registry.testing.TestOfyAndSql;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
|
||||
/** Tests for {@link SetDatabaseMigrationStateCommand}. */
|
||||
@@ -47,6 +48,11 @@ public class SetDatabaseMigrationStateCommandTest
|
||||
DatabaseMigrationStateSchedule.CACHE.invalidateAll();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
ofyTm().transact(() -> DatabaseMigrationStateSchedule.set(DEFAULT_TRANSITION_MAP.toValueMap()));
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_setsBasicSchedule() throws Exception {
|
||||
assertThat(DatabaseMigrationStateSchedule.get()).isEqualTo(DEFAULT_TRANSITION_MAP);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user