mirror of
https://github.com/google/nomulus
synced 2026-06-09 16:33:02 +00:00
Compare commits
32 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Lai Jiang
|
761ae612fd | ||
 gbrodman
|
e2fa60a9c6 | ||
 sarahcaseybot
|
b04dfbf740 | ||
 Weimin Yu
|
a1668ceafd | ||
|
Lai Jiang
|
406d49ac99 | ||
|
Weimin Yu
|
f7b82bc190 | ||
|
Lai Jiang
|
45c398149b | ||
|
gbrodman
|
183c0653fb | ||
|
gbrodman
|
fb002953c8 | ||
 Rachel Guan
|
a369e57e5c | ||
|
gbrodman
|
f32d80fb9d | ||
|
gbrodman
|
afa5a353f1 | ||
|
Rachel Guan
|
c4c5ac85da | ||
 Ben McIlwain
|
4d0078607f | ||
|
Rachel Guan
|
2b78433682 | ||
|
Weimin Yu
|
a0fcd02ed2 | ||
|
Rachel Guan
|
58e413af89 | ||
|
gbrodman
|
38c8e81690 | ||
|
Rachel Guan
|
3beb207fcc | ||
|
gbrodman
|
8cf88b7e18 | ||
|
gbrodman
|
6ec2e9501d | ||
|
sarahcaseybot
|
6849bf6914 | ||
|
gbrodman
|
34f3823960 | ||
|
gbrodman
|
bb5d2dcf0a | ||
|
sarahcaseybot
|
6ce0211537 | ||
|
Lai Jiang
|
676616a172 | ||
|
Weimin Yu
|
62c556cebf | ||
|
Ben McIlwain
|
535f84a912 | ||
|
sarahcaseybot
|
d283cf1c90 | ||
|
Rachel Guan
|
f5d344d5c9 | ||
|
Weimin Yu
|
61d029d955 | ||
|
Lai Jiang
|
2195ba90fa |
@@ -0,0 +1,4 @@
|
||||
To report a security issue, please use http://g.co/vulnz. We use
|
||||
http://g.co/vulnz for our intake, and do coordination and disclosure here on
|
||||
GitHub (including using GitHub Security Advisory). The Google Security Team will
|
||||
respond within 5 working days of your report on g.co/vulnz.
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -100,6 +100,15 @@ public class DateTimeUtils {
|
||||
return ZonedDateTime.ofInstant(instant, ZoneId.of(dateTime.getZone().getID()).normalized());
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a Joda {@link DateTime} object to an equivalent java.time {@link ZonedDateTime}
|
||||
* object.
|
||||
*/
|
||||
public static ZonedDateTime toZonedDateTime(DateTime dateTime, ZoneId zoneId) {
|
||||
java.time.Instant instant = java.time.Instant.ofEpochMilli(dateTime.getMillis());
|
||||
return ZonedDateTime.ofInstant(instant, zoneId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a java.time {@link ZonedDateTime} object to an equivalent Joda {@link DateTime}
|
||||
* object.
|
||||
|
||||
@@ -79,8 +79,8 @@ PRESUBMITS = {
|
||||
r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
|
||||
("java", "js", "soy", "sql", "py", "sh", "gradle"), {
|
||||
".git", "/build/", "/generated/", "/generated_tests/",
|
||||
"node_modules/", "JUnitBackports.java", "registrar_bin.",
|
||||
"registrar_dbg.", "google-java-format-diff.py",
|
||||
"node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java",
|
||||
"registrar_bin.", "registrar_dbg.", "google-java-format-diff.py",
|
||||
"nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
|
||||
}, REQUIRED):
|
||||
"File did not include the license header.",
|
||||
|
||||
+4
-2
@@ -74,8 +74,6 @@ def fragileTestPatterns = [
|
||||
"google/registry/cron/TldFanoutActionTest.*",
|
||||
// Test Datastore inexplicably aborts transaction.
|
||||
"google/registry/model/tmch/ClaimsListShardTest.*",
|
||||
// Creates large object (64MBytes), occasionally throws OOM error.
|
||||
"google/registry/model/server/KmsSecretRevisionTest.*",
|
||||
// Changes cache timeouts and for some reason appears to have contention
|
||||
// with other tests.
|
||||
"google/registry/whois/WhoisCommandFactoryTest.*",
|
||||
@@ -197,6 +195,7 @@ dependencies {
|
||||
compile deps['com.google.apis:google-api-services-groupssettings']
|
||||
compile deps['com.google.apis:google-api-services-monitoring']
|
||||
compile deps['com.google.apis:google-api-services-sheets']
|
||||
compile deps['com.google.apis:google-api-services-storage']
|
||||
testCompile deps['com.google.appengine:appengine-api-stubs']
|
||||
compile deps['com.google.appengine.tools:appengine-gcs-client']
|
||||
compile deps['com.google.appengine.tools:appengine-mapreduce']
|
||||
@@ -221,6 +220,8 @@ dependencies {
|
||||
gradleLint.ignore('unused-dependency') {
|
||||
compile deps['com.google.gwt:gwt-user']
|
||||
}
|
||||
compile deps['com.google.cloud:google-cloud-core']
|
||||
compile deps['com.google.cloud:google-cloud-storage']
|
||||
compile deps['com.google.http-client:google-http-client']
|
||||
compile deps['com.google.http-client:google-http-client-appengine']
|
||||
compile deps['com.google.http-client:google-http-client-jackson2']
|
||||
@@ -315,6 +316,7 @@ dependencies {
|
||||
annotationProcessor project(':processor')
|
||||
testAnnotationProcessor project(':processor')
|
||||
|
||||
testCompile deps['com.google.cloud:google-cloud-nio']
|
||||
testCompile deps['com.google.appengine:appengine-testing']
|
||||
testCompile deps['com.google.guava:guava-testlib']
|
||||
testCompile deps['com.google.monitoring-client:contrib']
|
||||
|
||||
@@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33
|
||||
com.google.dagger:dagger-spi:2.33
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotation:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_check_api:2.3.4
|
||||
com.google.errorprone:error_prone_core:2.3.4
|
||||
com.google.errorprone:error_prone_type_annotations:2.3.4
|
||||
com.google.errorprone:javac-shaded:9-dev-r4023-3
|
||||
com.google.googlejavaformat:google-java-format:1.5
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.protobuf:protobuf-java:3.4.0
|
||||
@@ -32,7 +32,7 @@ javax.inject:javax.inject:1
|
||||
javax.persistence:javax.persistence-api:2.2
|
||||
net.ltgt.gradle.incap:incap:0.2
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.checkerframework:dataflow:3.0.0
|
||||
org.checkerframework:javacutil:3.0.0
|
||||
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
|
||||
|
||||
@@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -215,7 +216,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -232,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -216,7 +217,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
@@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
@@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
||||
@@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33
|
||||
com.google.dagger:dagger-spi:2.33
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotation:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_check_api:2.3.4
|
||||
com.google.errorprone:error_prone_core:2.3.4
|
||||
com.google.errorprone:error_prone_type_annotations:2.3.4
|
||||
com.google.errorprone:javac-shaded:9-dev-r4023-3
|
||||
com.google.googlejavaformat:google-java-format:1.5
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.protobuf:protobuf-java:3.4.0
|
||||
@@ -30,7 +30,7 @@ javax.inject:javax.inject:1
|
||||
javax.persistence:javax.persistence-api:2.2
|
||||
net.ltgt.gradle.incap:incap:0.2
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.checkerframework:dataflow:3.0.0
|
||||
org.checkerframework:javacutil:3.0.0
|
||||
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
|
||||
|
||||
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -54,12 +54,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -84,10 +84,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -98,32 +98,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -135,9 +137,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -161,7 +163,7 @@ io.github.classgraph:classgraph:4.8.102
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
@@ -199,7 +201,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
net.java.dev.jna:jna:5.5.0
|
||||
@@ -307,7 +309,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -53,12 +53,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -83,10 +83,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -97,31 +97,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -133,9 +135,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -159,7 +161,7 @@ io.github.classgraph:classgraph:4.8.102
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
@@ -194,7 +196,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
net.java.dev.jna:jna:5.5.0
|
||||
@@ -301,7 +303,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
@@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.arnx:nashorn-promise:0.1.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -320,7 +322,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
@@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.arnx:nashorn-promise:0.1.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -321,7 +323,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -18,8 +18,10 @@ import static com.google.appengine.api.ThreadManager.currentRequestThreadFactory
|
||||
import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.BUCKET_OVERRIDE_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.FROM_TIME_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.TO_TIME_PARAM;
|
||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static java.util.concurrent.Executors.newFixedThreadPool;
|
||||
@@ -32,6 +34,9 @@ import google.registry.cron.CommitLogFanoutAction;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.Parameter;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.inject.Qualifier;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -75,6 +80,12 @@ public final class BackupModule {
|
||||
return extractRequiredDatetimeParameter(req, UPPER_CHECKPOINT_TIME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(BUCKET_OVERRIDE_PARAM)
|
||||
static Optional<String> provideBucketOverride(HttpServletRequest req) {
|
||||
return extractOptionalParameter(req, BUCKET_OVERRIDE_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(FROM_TIME_PARAM)
|
||||
static DateTime provideFromTime(HttpServletRequest req) {
|
||||
@@ -92,4 +103,9 @@ public final class BackupModule {
|
||||
static ListeningExecutorService provideListeningExecutorService() {
|
||||
return listeningDecorator(newFixedThreadPool(NUM_THREADS, currentRequestThreadFactory()));
|
||||
}
|
||||
|
||||
@Provides
|
||||
static ScheduledExecutorService provideScheduledExecutorService() {
|
||||
return Executors.newSingleThreadScheduledExecutor();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,18 +28,17 @@ import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static java.nio.channels.Channels.newOutputStream;
|
||||
import static java.util.Comparator.comparingLong;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
@@ -74,7 +73,8 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
|
||||
public static final String DIFF_FILE_PREFIX = "commit_diff_until_";
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
@Inject @Config("commitLogDiffExportBatchSize") int batchSize;
|
||||
@Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime;
|
||||
@@ -102,13 +102,13 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
|
||||
logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
|
||||
// Open an output channel to GCS, wrapped in a stream for convenience.
|
||||
try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
|
||||
new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString())
|
||||
.addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString())
|
||||
.addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size()))
|
||||
.build()))) {
|
||||
try (OutputStream gcsStream =
|
||||
gcsUtils.openOutputStream(
|
||||
BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
ImmutableMap.of(
|
||||
LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(),
|
||||
UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(),
|
||||
NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) {
|
||||
// Export the upper checkpoint itself.
|
||||
serializeEntity(upperCheckpoint, gcsStream);
|
||||
// If there are no manifests to export, stop early, now that we've written out the file with
|
||||
|
||||
@@ -15,30 +15,32 @@
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static google.registry.util.DateTimeUtils.latestOf;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.common.util.concurrent.ListeningExecutorService;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import google.registry.backup.BackupModule.Backups;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Provider;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Utility class to list commit logs diff files stored on GCS. */
|
||||
@@ -46,33 +48,49 @@ class GcsDiffFileLister {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
@Inject @Backups ListeningExecutorService executor;
|
||||
@Inject GcsDiffFileLister() {}
|
||||
/** Timeout for retrieving per-file information from GCS. */
|
||||
private static final Duration FILE_INFO_TIMEOUT_DURATION = Duration.ofMinutes(1);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Backups Provider<ListeningExecutorService> executorProvider;
|
||||
@Inject ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
@Inject
|
||||
GcsDiffFileLister() {}
|
||||
|
||||
/**
|
||||
* Traverses the sequence of diff files backwards from checkpointTime and inserts the file
|
||||
* metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
|
||||
* metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
|
||||
* more files are missing.
|
||||
*
|
||||
* @throws UncheckedExecutionException wrapping a {@link java.util.concurrent.TimeoutException} if
|
||||
* the GCS call fails to finish within one minute, or wrapping any other exception if
|
||||
* something else goes wrong.
|
||||
*/
|
||||
private boolean constructDiffSequence(
|
||||
Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
|
||||
String gcsBucket,
|
||||
Map<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo,
|
||||
DateTime fromTime,
|
||||
DateTime lastTime,
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence) {
|
||||
TreeMap<DateTime, BlobInfo> sequence) {
|
||||
DateTime checkpointTime = lastTime;
|
||||
while (isBeforeOrAt(fromTime, checkpointTime)) {
|
||||
GcsFileMetadata metadata;
|
||||
if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
|
||||
metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
|
||||
BlobInfo blobInfo;
|
||||
if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) {
|
||||
blobInfo =
|
||||
Futures.getUnchecked(
|
||||
Futures.withTimeout(
|
||||
upperBoundTimesToBlobInfo.get(checkpointTime),
|
||||
FILE_INFO_TIMEOUT_DURATION,
|
||||
scheduledExecutorService));
|
||||
} else {
|
||||
String filename = DIFF_FILE_PREFIX + checkpointTime;
|
||||
logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
|
||||
metadata = getMetadata(filename);
|
||||
blobInfo = getBlobInfo(gcsBucket, filename);
|
||||
|
||||
// If we hit a gap, quit.
|
||||
if (metadata == null) {
|
||||
if (blobInfo == null) {
|
||||
logger.atInfo().log(
|
||||
"Gap discovered in sequence terminating at %s, missing file: %s",
|
||||
sequence.lastKey(), filename);
|
||||
@@ -80,14 +98,15 @@ class GcsDiffFileLister {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sequence.put(checkpointTime, metadata);
|
||||
checkpointTime = getLowerBoundTime(metadata);
|
||||
sequence.put(checkpointTime, blobInfo);
|
||||
checkpointTime = getLowerBoundTime(blobInfo);
|
||||
}
|
||||
logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
|
||||
return true;
|
||||
}
|
||||
|
||||
ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
|
||||
ImmutableList<BlobInfo> listDiffFiles(
|
||||
String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) {
|
||||
logger.atInfo().log("Requested restore from time: %s", fromTime);
|
||||
if (toTime != null) {
|
||||
logger.atInfo().log(" Until time: %s", toTime);
|
||||
@@ -95,66 +114,74 @@ class GcsDiffFileLister {
|
||||
// List all of the diff files on GCS and build a map from each file's upper checkpoint time
|
||||
// (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
|
||||
// an upper checkpoint time > fromTime.
|
||||
TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
|
||||
= new TreeMap<>();
|
||||
Iterator<ListItem> listItems;
|
||||
TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
|
||||
String commitLogDiffPrefix = getCommitLogDiffPrefix(fromTime, toTime);
|
||||
ImmutableList<String> filenames;
|
||||
try {
|
||||
// TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
|
||||
listItems = gcsService.list(
|
||||
gcsBucket,
|
||||
new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
|
||||
filenames =
|
||||
gcsUtils.listFolderObjects(gcsBucket, commitLogDiffPrefix).stream()
|
||||
.map(s -> commitLogDiffPrefix + s)
|
||||
.collect(toImmutableList());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
DateTime lastUpperBoundTime = START_OF_TIME;
|
||||
while (listItems.hasNext()) {
|
||||
final String filename = listItems.next().getName();
|
||||
DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToMetadata.put(upperBoundTime, executor.submit(() -> getMetadata(filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
if (upperBoundTimesToMetadata.isEmpty()) {
|
||||
logger.atInfo().log("No files found");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
// Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
|
||||
// last file that we found) and finding its previous file until we either run out of files or
|
||||
// get to one that precedes "fromTime".
|
||||
//
|
||||
// GCS file listing is eventually consistent, so it's possible that we are missing a file. The
|
||||
// metadata of a file is sufficient to identify the preceding file, so if we start from the
|
||||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet = !constructDiffSequence(
|
||||
upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
// Verify that all of the elements in the original set are represented in the sequence. If we
|
||||
// find anything that's not represented, construct a sequence for it.
|
||||
boolean checkForMoreExtraDiffs = true; // Always loop at least once.
|
||||
while (checkForMoreExtraDiffs) {
|
||||
checkForMoreExtraDiffs = false;
|
||||
for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
|
||||
if (!isInRange(key, fromTime, toTime)) {
|
||||
break;
|
||||
}
|
||||
if (!sequence.containsKey(key)) {
|
||||
constructDiffSequence(upperBoundTimesToMetadata, fromTime, key, sequence);
|
||||
checkForMoreExtraDiffs = true;
|
||||
inconsistentFileSet = true;
|
||||
break;
|
||||
TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
|
||||
ListeningExecutorService executor = executorProvider.get();
|
||||
try {
|
||||
for (String filename : filenames) {
|
||||
String strippedFilename = filename.replaceFirst(DIFF_FILE_PREFIX, "");
|
||||
DateTime upperBoundTime = DateTime.parse(strippedFilename);
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToBlobInfo.put(
|
||||
upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (upperBoundTimesToBlobInfo.isEmpty()) {
|
||||
logger.atInfo().log("No files found");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
checkState(
|
||||
!inconsistentFileSet,
|
||||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
// Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e.
|
||||
// the last file that we found) and finding its previous file until we either run out of files
|
||||
// or get to one that precedes "fromTime".
|
||||
//
|
||||
// GCS file listing is eventually consistent, so it's possible that we are missing a file. The
|
||||
// metadata of a file is sufficient to identify the preceding file, so if we start from the
|
||||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet =
|
||||
!constructDiffSequence(
|
||||
gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
// Verify that all of the elements in the original set are represented in the sequence. If we
|
||||
// find anything that's not represented, construct a sequence for it.
|
||||
boolean checkForMoreExtraDiffs = true; // Always loop at least once.
|
||||
while (checkForMoreExtraDiffs) {
|
||||
checkForMoreExtraDiffs = false;
|
||||
for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) {
|
||||
if (!isInRange(key, fromTime, toTime)) {
|
||||
break;
|
||||
}
|
||||
if (!sequence.containsKey(key)) {
|
||||
constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
|
||||
checkForMoreExtraDiffs = true;
|
||||
inconsistentFileSet = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checkState(
|
||||
!inconsistentFileSet,
|
||||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
} finally {
|
||||
executor.shutdown();
|
||||
}
|
||||
|
||||
logger.atInfo().log(
|
||||
"Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
|
||||
@@ -171,15 +198,43 @@ class GcsDiffFileLister {
|
||||
return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end));
|
||||
}
|
||||
|
||||
private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
|
||||
return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
private DateTime getLowerBoundTime(BlobInfo blobInfo) {
|
||||
return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
}
|
||||
|
||||
private GcsFileMetadata getMetadata(String filename) {
|
||||
try {
|
||||
return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
private BlobInfo getBlobInfo(String gcsBucket, String filename) {
|
||||
return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a prefix guaranteed to cover all commit log diff files in the given range.
|
||||
*
|
||||
* <p>The listObjects call can be fairly slow if we search over many thousands or tens of
|
||||
* thousands of files, so we restrict the search space. The commit logs have a file format of
|
||||
* "commit_diff_until_2021-05-11T06:48:00.070Z" so we can often filter down as far as the hour.
|
||||
*
|
||||
* <p>Here, we get the longest prefix possible based on which fields (year, month, day, hour) the
|
||||
* times in question have in common.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static String getCommitLogDiffPrefix(DateTime from, @Nullable DateTime to) {
|
||||
StringBuilder result = new StringBuilder(DIFF_FILE_PREFIX);
|
||||
if (to == null || from.getYear() != to.getYear()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(from.getYear()).append('-');
|
||||
if (from.getMonthOfYear() != to.getMonthOfYear()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02d-", from.getMonthOfYear()));
|
||||
if (from.getDayOfMonth() != to.getDayOfMonth()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02dT", from.getDayOfMonth()));
|
||||
if (from.getHourOfDay() != to.getHourOfDay()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02d:", from.getHourOfDay()));
|
||||
return result.toString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,16 +20,19 @@ import static google.registry.backup.RestoreCommitLogsAction.DRY_RUN_PARAM;
|
||||
import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.joda.time.Duration.standardHours;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.Key;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
|
||||
@@ -50,7 +53,6 @@ import google.registry.util.Clock;
|
||||
import google.registry.util.RequestStatusChecker;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
@@ -69,19 +71,22 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
static final String PATH = "/_dr/task/replayCommitLogsToSql";
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final int BLOCK_SIZE =
|
||||
1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
|
||||
private static final Duration LEASE_LENGTH = standardHours(1);
|
||||
// Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
|
||||
// request timeouts
|
||||
private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject Response response;
|
||||
@Inject RequestStatusChecker requestStatusChecker;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
@Inject Clock clock;
|
||||
|
||||
@Inject
|
||||
@Config("commitLogGcsBucket")
|
||||
String gcsBucket;
|
||||
|
||||
/** If true, will exit after logging the commit log files that would otherwise be replayed. */
|
||||
@Inject
|
||||
@Parameter(DRY_RUN_PARAM)
|
||||
@@ -117,26 +122,15 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
}
|
||||
try {
|
||||
logger.atInfo().log("Beginning replay of commit logs.");
|
||||
ImmutableList<GcsFileMetadata> commitLogFiles = getFilesToReplay();
|
||||
String resultMessage;
|
||||
if (dryRun) {
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
ImmutableList<String> filenames =
|
||||
commitLogFiles.stream()
|
||||
.limit(10)
|
||||
.map(file -> file.getFilename().getObjectName())
|
||||
.collect(toImmutableList());
|
||||
String dryRunMessage =
|
||||
"Running in dry-run mode; would have processed %d files. They are (limit 10):\n"
|
||||
+ Joiner.on('\n').join(filenames);
|
||||
response.setPayload(dryRunMessage);
|
||||
logger.atInfo().log(dryRunMessage);
|
||||
resultMessage = executeDryRun();
|
||||
} else {
|
||||
replayFiles(commitLogFiles);
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
String message = "ReplayCommitLogsToSqlAction completed successfully.";
|
||||
response.setPayload(message);
|
||||
logger.atInfo().log(message);
|
||||
resultMessage = replayFiles();
|
||||
}
|
||||
response.setStatus(SC_OK);
|
||||
response.setPayload(resultMessage);
|
||||
logger.atInfo().log(resultMessage);
|
||||
} catch (Throwable t) {
|
||||
String message = "Errored out replaying files.";
|
||||
logger.atSevere().withCause(t).log(message);
|
||||
@@ -147,60 +141,81 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
private ImmutableList<GcsFileMetadata> getFilesToReplay() {
|
||||
private String executeDryRun() {
|
||||
// Start at the first millisecond we haven't seen yet
|
||||
DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
logger.atInfo().log("Starting replay from: %s.", fromTime);
|
||||
// If there's an inconsistent file set, this will throw IllegalStateException and the job
|
||||
// will try later -- this is likely because an export hasn't finished yet.
|
||||
ImmutableList<GcsFileMetadata> commitLogFiles =
|
||||
diffLister.listDiffFiles(fromTime, /* current time */ null);
|
||||
logger.atInfo().log("Found %d new commit log files to process.", commitLogFiles.size());
|
||||
return commitLogFiles;
|
||||
DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
// Search through the end of the hour
|
||||
DateTime searchEndTime =
|
||||
searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
|
||||
ImmutableList<String> fileBatch =
|
||||
diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime).stream()
|
||||
.map(BlobInfo::getName)
|
||||
.collect(toImmutableList());
|
||||
return String.format(
|
||||
"Running in dry-run mode, the first set of commit log files processed would be from "
|
||||
+ "searching from %s to %s and would contain %d file(s). They are (limit 10): \n%s",
|
||||
searchStartTime,
|
||||
searchEndTime,
|
||||
fileBatch.size(),
|
||||
fileBatch.stream().limit(10).collect(toImmutableList()));
|
||||
}
|
||||
|
||||
private void replayFiles(ImmutableList<GcsFileMetadata> commitLogFiles) {
|
||||
private String replayFiles() {
|
||||
DateTime replayTimeoutTime = clock.nowUtc().plus(REPLAY_TIMEOUT_DURATION);
|
||||
int processedFiles = 0;
|
||||
for (GcsFileMetadata metadata : commitLogFiles) {
|
||||
// One transaction per GCS file
|
||||
jpaTm().transact(() -> processFile(metadata));
|
||||
processedFiles++;
|
||||
if (clock.nowUtc().isAfter(replayTimeoutTime)) {
|
||||
logger.atInfo().log(
|
||||
"Reached max execution time after replaying %d files, leaving %d files for next run.",
|
||||
processedFiles, commitLogFiles.size() - processedFiles);
|
||||
return;
|
||||
DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
int filesProcessed = 0;
|
||||
// Starting from one millisecond after the last file we processed, search for and import files
|
||||
// one hour at a time until we catch up to the current time or we hit the replay timeout (in
|
||||
// which case the next run will pick up from where we leave off).
|
||||
//
|
||||
// We use hour-long batches because GCS supports filename prefix-based searches.
|
||||
while (true) {
|
||||
if (isAtOrAfter(clock.nowUtc(), replayTimeoutTime)) {
|
||||
return String.format(
|
||||
"Reached max execution time after replaying %d file(s).", filesProcessed);
|
||||
}
|
||||
if (isBeforeOrAt(clock.nowUtc(), searchStartTime)) {
|
||||
return String.format(
|
||||
"Caught up to current time after replaying %d file(s).", filesProcessed);
|
||||
}
|
||||
// Search through the end of the hour
|
||||
DateTime searchEndTime =
|
||||
searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
|
||||
ImmutableList<BlobInfo> fileBatch =
|
||||
diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime);
|
||||
if (fileBatch.isEmpty()) {
|
||||
logger.atInfo().log(
|
||||
"No remaining files found in hour %s, continuing search in the next hour.",
|
||||
searchStartTime.toString("yyyy-MM-dd HH"));
|
||||
}
|
||||
for (BlobInfo file : fileBatch) {
|
||||
processFile(file);
|
||||
filesProcessed++;
|
||||
if (clock.nowUtc().isAfter(replayTimeoutTime)) {
|
||||
return String.format(
|
||||
"Reached max execution time after replaying %d file(s).", filesProcessed);
|
||||
}
|
||||
}
|
||||
searchStartTime = searchEndTime.plusMillis(1);
|
||||
}
|
||||
logger.atInfo().log("Replayed %d commit log files to SQL successfully.", processedFiles);
|
||||
}
|
||||
|
||||
private void processFile(GcsFileMetadata metadata) {
|
||||
logger.atInfo().log(
|
||||
"Processing commit log file %s of size %d B.",
|
||||
metadata.getFilename(), metadata.getLength());
|
||||
try (InputStream input =
|
||||
Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
private void processFile(BlobInfo metadata) {
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
// Load and process the Datastore transactions one at a time
|
||||
ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
|
||||
CommitLogImports.loadEntitiesByTransaction(input);
|
||||
logger.atInfo().log(
|
||||
"Replaying %d transactions from commit log file %s.",
|
||||
allTransactions.size(), metadata.getFilename());
|
||||
allTransactions.forEach(this::replayTransaction);
|
||||
allTransactions.forEach(
|
||||
transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
|
||||
// if we succeeded, set the last-seen time
|
||||
DateTime checkpoint =
|
||||
DateTime.parse(
|
||||
metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
|
||||
SqlReplayCheckpoint.set(checkpoint);
|
||||
DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(checkpoint));
|
||||
logger.atInfo().log(
|
||||
"Replayed %d transactions from commit log file %s.",
|
||||
allTransactions.size(), metadata.getFilename());
|
||||
"Replayed %d transactions from commit log file %s with size %d B.",
|
||||
allTransactions.size(), metadata.getName(), metadata.getSize());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(
|
||||
"Errored out while replaying commit log file " + metadata.getFilename(), e);
|
||||
"Errored out while replaying commit log file " + metadata.getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,20 +234,25 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
|
||||
private void handleEntityPut(Entity entity) {
|
||||
Object ofyPojo = auditedOfy().toPojo(entity);
|
||||
if (ofyPojo instanceof DatastoreEntity) {
|
||||
DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
|
||||
datastoreEntity
|
||||
.toSqlEntity()
|
||||
.ifPresent(
|
||||
sqlEntity -> {
|
||||
ReplaySpecializer.beforeSqlSave(sqlEntity);
|
||||
jpaTm().put(sqlEntity);
|
||||
});
|
||||
} else {
|
||||
// this should never happen, but we shouldn't fail on it
|
||||
logger.atSevere().log(
|
||||
"%s does not implement DatastoreEntity, which is necessary for SQL replay.",
|
||||
ofyPojo.getClass());
|
||||
try {
|
||||
if (ofyPojo instanceof DatastoreEntity) {
|
||||
DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
|
||||
datastoreEntity
|
||||
.toSqlEntity()
|
||||
.ifPresent(
|
||||
sqlEntity -> {
|
||||
sqlEntity.beforeSqlSaveOnReplay();
|
||||
jpaTm().put(sqlEntity);
|
||||
});
|
||||
} else {
|
||||
// this should never happen, but we shouldn't fail on it
|
||||
logger.atSevere().log(
|
||||
"%s does not implement DatastoreEntity, which is necessary for SQL replay.",
|
||||
ofyPojo.getClass());
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().log("Error when replaying object %s", ofyPojo);
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -248,13 +268,18 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
"Skipping SQL delete for kind %s since it is not convertible.", key.getKind());
|
||||
return;
|
||||
}
|
||||
Class<?> entityClass = entityVKey.getKind();
|
||||
// Delete the key iff the class represents a JPA entity that is replicated
|
||||
if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
|
||||
&& !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
|
||||
&& entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
|
||||
ReplaySpecializer.beforeSqlDelete(entityVKey);
|
||||
jpaTm().delete(entityVKey);
|
||||
try {
|
||||
Class<?> entityClass = entityVKey.getKind();
|
||||
// Delete the key iff the class represents a JPA entity that is replicated
|
||||
if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
|
||||
&& !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
|
||||
&& entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
|
||||
ReplaySpecializer.beforeSqlDelete(entityVKey);
|
||||
jpaTm().delete(entityVKey);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().log("Error when deleting key %s", entityVKey);
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import com.google.appengine.api.datastore.DatastoreService;
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.EntityTranslator;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
import com.google.common.collect.Streams;
|
||||
@@ -33,7 +33,9 @@ import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Result;
|
||||
import com.googlecode.objectify.util.ResultNow;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
@@ -46,10 +48,10 @@ import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Retrier;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
import javax.inject.Inject;
|
||||
@@ -66,43 +68,55 @@ public class RestoreCommitLogsAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
static final int BLOCK_SIZE = 1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
|
||||
public static final String PATH = "/_dr/task/restoreCommitLogs";
|
||||
static final String DRY_RUN_PARAM = "dryRun";
|
||||
static final String FROM_TIME_PARAM = "fromTime";
|
||||
static final String TO_TIME_PARAM = "toTime";
|
||||
static final String BUCKET_OVERRIDE_PARAM = "gcsBucket";
|
||||
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject @Parameter(DRY_RUN_PARAM) boolean dryRun;
|
||||
@Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime;
|
||||
@Inject @Parameter(TO_TIME_PARAM) DateTime toTime;
|
||||
|
||||
@Inject
|
||||
@Parameter(BUCKET_OVERRIDE_PARAM)
|
||||
Optional<String> gcsBucketOverride;
|
||||
|
||||
@Inject DatastoreService datastoreService;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
|
||||
@Inject
|
||||
@Config("commitLogGcsBucket")
|
||||
String defaultGcsBucket;
|
||||
|
||||
@Inject Retrier retrier;
|
||||
@Inject RestoreCommitLogsAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
checkArgument(
|
||||
RegistryEnvironment.get() == RegistryEnvironment.ALPHA
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.CRASH
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
|
||||
"DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
|
||||
!FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
|
||||
"DO NOT RUN IN PRODUCTION OR SANDBOX.");
|
||||
if (dryRun) {
|
||||
logger.atInfo().log("Running in dryRun mode");
|
||||
}
|
||||
List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
|
||||
String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
|
||||
logger.atInfo().log("Restoring from %s.", gcsBucket);
|
||||
List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
|
||||
if (diffFiles.isEmpty()) {
|
||||
logger.atInfo().log("Nothing to restore");
|
||||
return;
|
||||
}
|
||||
Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
|
||||
CommitLogCheckpoint lastCheckpoint = null;
|
||||
for (GcsFileMetadata metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
|
||||
try (InputStream input = Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
for (BlobInfo metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getName());
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
PeekingIterator<ImmutableObject> commitLogs =
|
||||
peekingIterator(createDeserializingIterator(input, true));
|
||||
lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
|
||||
|
||||
@@ -22,7 +22,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.persistence.PersistenceModule.SchemaManagerConnection;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
@@ -48,22 +48,18 @@ import javax.inject.Inject;
|
||||
public class WipeOutCloudSqlAction implements Runnable {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
// As a short-lived class, hardcode allowed projects here instead of using config files.
|
||||
private static final ImmutableSet<String> ALLOWED_PROJECTS =
|
||||
ImmutableSet.of("domain-registry-qa");
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
private final String projectId;
|
||||
private final Supplier<Connection> connectionSupplier;
|
||||
private final Response response;
|
||||
private final Retrier retrier;
|
||||
|
||||
@Inject
|
||||
WipeOutCloudSqlAction(
|
||||
@Config("projectId") String projectId,
|
||||
@SchemaManagerConnection Supplier<Connection> connectionSupplier,
|
||||
Response response,
|
||||
Retrier retrier) {
|
||||
this.projectId = projectId;
|
||||
this.connectionSupplier = connectionSupplier;
|
||||
this.response = response;
|
||||
this.retrier = retrier;
|
||||
@@ -73,9 +69,9 @@ public class WipeOutCloudSqlAction implements Runnable {
|
||||
public void run() {
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
|
||||
if (!ALLOWED_PROJECTS.contains(projectId)) {
|
||||
if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
|
||||
response.setStatus(SC_FORBIDDEN);
|
||||
response.setPayload("Wipeout is not allowed in " + projectId);
|
||||
response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -90,11 +86,11 @@ public class WipeOutCloudSqlAction implements Runnable {
|
||||
},
|
||||
e -> !(e instanceof SQLException));
|
||||
response.setStatus(SC_OK);
|
||||
response.setPayload("Wiped out Cloud SQL in " + projectId);
|
||||
response.setPayload("Wiped out Cloud SQL in " + RegistryEnvironment.get());
|
||||
} catch (RuntimeException e) {
|
||||
logger.atSevere().withCause(e).log("Failed to wipe out Cloud SQL data.");
|
||||
response.setStatus(SC_INTERNAL_SERVER_ERROR);
|
||||
response.setPayload("Failed to wipe out Cloud SQL in " + projectId);
|
||||
response.setPayload("Failed to wipe out Cloud SQL in " + RegistryEnvironment.get());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
@@ -49,9 +50,8 @@ public class WipeoutDatastoreAction implements Runnable {
|
||||
|
||||
private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";
|
||||
|
||||
// As a short-lived class, hardcode allowed projects here instead of using config files.
|
||||
private static final ImmutableSet<String> ALLOWED_PROJECTS =
|
||||
ImmutableSet.of("domain-registry-qa");
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
private final String projectId;
|
||||
private final String jobRegion;
|
||||
@@ -80,9 +80,9 @@ public class WipeoutDatastoreAction implements Runnable {
|
||||
public void run() {
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
|
||||
if (!ALLOWED_PROJECTS.contains(projectId)) {
|
||||
if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
|
||||
response.setStatus(SC_FORBIDDEN);
|
||||
response.setPayload("Wipeout is not allowed in " + projectId);
|
||||
response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.initsql;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
|
||||
import com.google.api.services.cloudkms.v1.model.DecryptRequest;
|
||||
import com.google.common.base.Strings;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.keyring.kms.KmsConnection;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Base64;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Decrypts data using Cloud KMS, with the same crypto key with which Cloud SQL credential files on
|
||||
* GCS was encrypted. See {@link BackupPaths#getCloudSQLCredentialFilePatterns} for more
|
||||
* information.
|
||||
*/
|
||||
public class CloudSqlCredentialDecryptor {
|
||||
|
||||
private static final String CRYPTO_KEY_NAME = "nomulus-tool-key";
|
||||
private final KmsConnection kmsConnection;
|
||||
|
||||
@Inject
|
||||
CloudSqlCredentialDecryptor(@Config("beamKmsConnection") KmsConnection kmsConnection) {
|
||||
this.kmsConnection = kmsConnection;
|
||||
}
|
||||
|
||||
public String decrypt(String data) {
|
||||
checkArgument(!Strings.isNullOrEmpty(data), "Null or empty data.");
|
||||
byte[] ciphertext = Base64.getDecoder().decode(data);
|
||||
// Re-encode for Cloud KMS JSON REST API, invoked through kmsConnection.
|
||||
String urlSafeCipherText = new DecryptRequest().encodeCiphertext(ciphertext).getCiphertext();
|
||||
return new String(
|
||||
kmsConnection.decrypt(CRYPTO_KEY_NAME, urlSafeCipherText), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.initsql;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Information needed to connect to a database, including JDBC URL, user name, password, and in the
|
||||
* case of Cloud SQL, the database instance's name.
|
||||
*/
|
||||
@AutoValue
|
||||
abstract class SqlAccessInfo {
|
||||
|
||||
abstract String jdbcUrl();
|
||||
|
||||
abstract String user();
|
||||
|
||||
abstract String password();
|
||||
|
||||
abstract Optional<String> cloudSqlInstanceName();
|
||||
|
||||
public static SqlAccessInfo createCloudSqlAccessInfo(
|
||||
String sqlInstanceName, String username, String password) {
|
||||
return new AutoValue_SqlAccessInfo(
|
||||
"jdbc:postgresql://google/postgres", username, password, Optional.of(sqlInstanceName));
|
||||
}
|
||||
|
||||
public static SqlAccessInfo createLocalSqlAccessInfo(
|
||||
String jdbcUrl, String username, String password) {
|
||||
return new AutoValue_SqlAccessInfo(jdbcUrl, username, password, Optional.empty());
|
||||
}
|
||||
}
|
||||
@@ -14,18 +14,29 @@
|
||||
|
||||
package google.registry.beam.invoicing;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.beam.BeamUtils.getQueryFromFile;
|
||||
import static org.apache.beam.sdk.values.TypeDescriptors.strings;
|
||||
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.beam.common.RegistryJpaIO.Read;
|
||||
import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey;
|
||||
import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.reporting.billing.BillingModule;
|
||||
import google.registry.util.DateTimeUtils;
|
||||
import google.registry.util.DomainNameUtils;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.LocalTime;
|
||||
import java.time.YearMonth;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Pattern;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.PipelineResult;
|
||||
import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
@@ -58,6 +69,9 @@ public class InvoicingPipeline implements Serializable {
|
||||
private static final DateTimeFormatter TIMESTAMP_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS");
|
||||
|
||||
private static final Pattern SQL_COMMENT_REGEX =
|
||||
Pattern.compile("^\\s*--.*\\n", Pattern.MULTILINE);
|
||||
|
||||
private final InvoicingPipelineOptions options;
|
||||
|
||||
InvoicingPipeline(InvoicingPipelineOptions options) {
|
||||
@@ -71,21 +85,60 @@ public class InvoicingPipeline implements Serializable {
|
||||
}
|
||||
|
||||
void setupPipeline(Pipeline pipeline) {
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
PCollection<BillingEvent> billingEvents =
|
||||
pipeline.apply(
|
||||
"Read BillingEvents from Bigquery",
|
||||
BigQueryIO.read(BillingEvent::parseFromRecord)
|
||||
.fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
|
||||
.withCoder(SerializableCoder.of(BillingEvent.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
options.getDatabase().equals("DATASTORE")
|
||||
? readFromBigQuery(options, pipeline)
|
||||
: readFromCloudSql(options, pipeline);
|
||||
|
||||
saveInvoiceCsv(billingEvents, options);
|
||||
|
||||
saveDetailedCsv(billingEvents, options);
|
||||
}
|
||||
|
||||
static PCollection<BillingEvent> readFromBigQuery(
|
||||
InvoicingPipelineOptions options, Pipeline pipeline) {
|
||||
return pipeline.apply(
|
||||
"Read BillingEvents from Bigquery",
|
||||
BigQueryIO.read(BillingEvent::parseFromRecord)
|
||||
.fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
|
||||
.withCoder(SerializableCoder.of(BillingEvent.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
}
|
||||
|
||||
static PCollection<BillingEvent> readFromCloudSql(
|
||||
InvoicingPipelineOptions options, Pipeline pipeline) {
|
||||
Read<Object[], BillingEvent> read =
|
||||
RegistryJpaIO.read(
|
||||
makeCloudSqlQuery(options.getYearMonth()), false, InvoicingPipeline::parseRow);
|
||||
|
||||
return pipeline.apply("Read BillingEvents from Cloud SQL", read);
|
||||
}
|
||||
|
||||
private static BillingEvent parseRow(Object[] row) {
|
||||
google.registry.model.billing.BillingEvent.OneTime oneTime =
|
||||
(google.registry.model.billing.BillingEvent.OneTime) row[0];
|
||||
Registrar registrar = (Registrar) row[1];
|
||||
return BillingEvent.create(
|
||||
oneTime.getId(),
|
||||
DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
|
||||
DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
|
||||
registrar.getClientId(),
|
||||
registrar.getBillingIdentifier().toString(),
|
||||
registrar.getPoNumber().orElse(""),
|
||||
DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),
|
||||
oneTime.getReason().toString(),
|
||||
oneTime.getTargetId(),
|
||||
oneTime.getDomainRepoId(),
|
||||
Optional.ofNullable(oneTime.getPeriodYears()).orElse(0),
|
||||
oneTime.getCost().getCurrencyUnit().toString(),
|
||||
oneTime.getCost().getAmount().doubleValue(),
|
||||
String.join(
|
||||
" ", oneTime.getFlags().stream().map(Flag::toString).collect(toImmutableSet())));
|
||||
}
|
||||
|
||||
/** Transform that converts a {@code BillingEvent} into an invoice CSV row. */
|
||||
private static class GenerateInvoiceRows
|
||||
extends PTransform<PCollection<BillingEvent>, PCollection<String>> {
|
||||
@@ -171,6 +224,21 @@ public class InvoicingPipeline implements Serializable {
|
||||
.build();
|
||||
}
|
||||
|
||||
/** Create the Cloud SQL query for a given yearMonth at runtime. */
|
||||
static String makeCloudSqlQuery(String yearMonth) {
|
||||
YearMonth endMonth = YearMonth.parse(yearMonth).plusMonths(1);
|
||||
String queryWithComments =
|
||||
SqlTemplate.create(
|
||||
getQueryFromFile(InvoicingPipeline.class, "cloud_sql_billing_events.sql"))
|
||||
.put("FIRST_TIMESTAMP_OF_MONTH", yearMonth.concat("-01"))
|
||||
.put(
|
||||
"LAST_TIMESTAMP_OF_MONTH",
|
||||
String.format("%d-%d-01", endMonth.getYear(), endMonth.getMonthValue()))
|
||||
.build();
|
||||
// Remove the comments from the query string
|
||||
return SQL_COMMENT_REGEX.matcher(queryWithComments).replaceAll("");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
PipelineOptionsFactory.register(InvoicingPipelineOptions.class);
|
||||
InvoicingPipelineOptions options =
|
||||
|
||||
@@ -30,6 +30,11 @@ public interface InvoicingPipelineOptions extends RegistryPipelineOptions {
|
||||
|
||||
void setInvoiceFilePrefix(String value);
|
||||
|
||||
@Description("The database to read data from.")
|
||||
String getDatabase();
|
||||
|
||||
void setDatabase(String value);
|
||||
|
||||
@Description("The GCS bucket URL for invoices and detailed reports to be uploaded.")
|
||||
String getBillingBucketUrl();
|
||||
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.rde;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.rde.RdeMode;
|
||||
import google.registry.model.rde.RdeNamingUtils;
|
||||
import google.registry.model.rde.RdeRevision;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.Ghostryde;
|
||||
import google.registry.rde.PendingDeposit;
|
||||
import google.registry.rde.RdeCounter;
|
||||
import google.registry.rde.RdeMarshaller;
|
||||
import google.registry.rde.RdeResourceType;
|
||||
import google.registry.rde.RdeUtil;
|
||||
import google.registry.tldconfig.idn.IdnTableEnum;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeader;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
|
||||
import google.registry.xml.ValidationMode;
|
||||
import google.registry.xml.XmlException;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.security.Security;
|
||||
import java.util.Optional;
|
||||
import org.apache.beam.sdk.options.PipelineOptions;
|
||||
import org.apache.beam.sdk.transforms.DoFn;
|
||||
import org.apache.beam.sdk.transforms.PTransform;
|
||||
import org.apache.beam.sdk.transforms.ParDo;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.apache.beam.sdk.values.PDone;
|
||||
import org.bouncycastle.jce.provider.BouncyCastleProvider;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
public class RdeIO {
|
||||
|
||||
@AutoValue
|
||||
abstract static class Write
|
||||
extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
|
||||
|
||||
abstract GcsUtils gcsUtils();
|
||||
|
||||
abstract String rdeBucket();
|
||||
|
||||
// It's OK to return a primitive array because we are only using it to construct the
|
||||
// PGPPublicKey, which is not serializable.
|
||||
@SuppressWarnings("mutable")
|
||||
abstract byte[] stagingKeyBytes();
|
||||
|
||||
abstract ValidationMode validationMode();
|
||||
|
||||
static Builder builder() {
|
||||
return new AutoValue_RdeIO_Write.Builder();
|
||||
}
|
||||
|
||||
@AutoValue.Builder
|
||||
abstract static class Builder {
|
||||
abstract Builder setGcsUtils(GcsUtils gcsUtils);
|
||||
|
||||
abstract Builder setRdeBucket(String value);
|
||||
|
||||
abstract Builder setStagingKeyBytes(byte[] value);
|
||||
|
||||
abstract Builder setValidationMode(ValidationMode value);
|
||||
|
||||
abstract Write build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PDone expand(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
|
||||
input
|
||||
.apply(
|
||||
"Write to GCS",
|
||||
ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
|
||||
.apply("Update cursors", ParDo.of(new CursorUpdater()));
|
||||
return PDone.in(input.getPipeline());
|
||||
}
|
||||
}
|
||||
|
||||
private static class RdeWriter
|
||||
extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private final GcsUtils gcsUtils;
|
||||
private final String rdeBucket;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final RdeMarshaller marshaller;
|
||||
|
||||
protected RdeWriter(
|
||||
GcsUtils gcsUtils,
|
||||
String rdeBucket,
|
||||
byte[] stagingKeyBytes,
|
||||
ValidationMode validationMode) {
|
||||
this.gcsUtils = gcsUtils;
|
||||
this.rdeBucket = rdeBucket;
|
||||
this.stagingKeyBytes = stagingKeyBytes;
|
||||
this.marshaller = new RdeMarshaller(validationMode);
|
||||
}
|
||||
|
||||
@Setup
|
||||
public void setup() {
|
||||
Security.addProvider(new BouncyCastleProvider());
|
||||
}
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(
|
||||
@Element KV<PendingDeposit, Iterable<DepositFragment>> kv,
|
||||
PipelineOptions options,
|
||||
OutputReceiver<KV<PendingDeposit, Integer>> outputReceiver) {
|
||||
PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
|
||||
PendingDeposit key = kv.getKey();
|
||||
Iterable<DepositFragment> fragments = kv.getValue();
|
||||
RdeCounter counter = new RdeCounter();
|
||||
|
||||
// Determine some basic things about the deposit.
|
||||
final RdeMode mode = key.mode();
|
||||
final String tld = key.tld();
|
||||
final DateTime watermark = key.watermark();
|
||||
final int revision =
|
||||
Optional.ofNullable(key.revision())
|
||||
.orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
|
||||
String id = RdeUtil.timestampToId(watermark);
|
||||
String prefix = options.getJobName();
|
||||
String basename = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
|
||||
if (key.manual()) {
|
||||
checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
|
||||
prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
|
||||
} else {
|
||||
prefix = prefix + "/" + basename;
|
||||
}
|
||||
BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
|
||||
// This file will contain the byte length (ASCII) of the raw unencrypted XML.
|
||||
//
|
||||
// This is necessary because RdeUploadAction creates a tar file which requires that the length
|
||||
// be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
|
||||
// length, so we just save it separately.
|
||||
BlobId xmlLengthFilename = BlobId.of(rdeBucket, prefix + ".xml.length");
|
||||
BlobId reportFilename = BlobId.of(rdeBucket, prefix + "-report.xml.ghostryde");
|
||||
|
||||
// These variables will be populated as we write the deposit XML and used for other files.
|
||||
boolean failed = false;
|
||||
XjcRdeHeader header;
|
||||
|
||||
// Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
|
||||
|
||||
logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
|
||||
Writer output = new OutputStreamWriter(ghostrydeEncoder, UTF_8)) {
|
||||
|
||||
// Output the top portion of the XML document.
|
||||
output.write(marshaller.makeHeader(id, watermark, RdeResourceType.getUris(mode), revision));
|
||||
|
||||
// Output XML fragments while counting them.
|
||||
for (DepositFragment fragment : fragments) {
|
||||
if (!fragment.xml().isEmpty()) {
|
||||
output.write(fragment.xml());
|
||||
counter.increment(fragment.type());
|
||||
}
|
||||
if (!fragment.error().isEmpty()) {
|
||||
failed = true;
|
||||
logger.atSevere().log("Fragment error: %s", fragment.error());
|
||||
}
|
||||
}
|
||||
|
||||
// Don't write the IDN elements for BRDA.
|
||||
if (mode == RdeMode.FULL) {
|
||||
for (IdnTableEnum idn : IdnTableEnum.values()) {
|
||||
output.write(marshaller.marshalIdn(idn.getTable()));
|
||||
counter.increment(RdeResourceType.IDN);
|
||||
}
|
||||
}
|
||||
|
||||
// Output XML that says how many resources were emitted.
|
||||
header = counter.makeHeader(tld, mode);
|
||||
output.write(marshaller.marshalOrDie(new XjcRdeHeaderElement(header)));
|
||||
|
||||
// Output the bottom of the XML document.
|
||||
output.write(marshaller.makeFooter());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
// If an entity was broken, abort after writing as much logs/deposit data as possible.
|
||||
verify(!failed, "RDE staging failed for TLD %s", tld);
|
||||
|
||||
// Write a tiny XML file to GCS containing some information about the deposit.
|
||||
//
|
||||
// This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
|
||||
if (mode == RdeMode.FULL) {
|
||||
logger.atInfo().log("Writing %s", reportFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
|
||||
counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
|
||||
} catch (IOException | XmlException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
// Now that we're done, output roll the cursor forward.
|
||||
if (key.manual()) {
|
||||
logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task");
|
||||
} else {
|
||||
outputReceiver.output(KV.of(key, revision));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(@Element KV<PendingDeposit, Integer> input) {
|
||||
tm().transact(
|
||||
() -> {
|
||||
PendingDeposit key = input.getKey();
|
||||
int revision = input.getValue();
|
||||
Registry registry = Registry.get(key.tld());
|
||||
Optional<Cursor> cursor =
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
tm().loadByKeyIfPresent(
|
||||
Cursor.createVKey(key.cursor(), registry.getTldStr())));
|
||||
DateTime position = getCursorTimeOrStartOfTime(cursor);
|
||||
checkState(key.interval() != null, "Interval must be present");
|
||||
DateTime newPosition = key.watermark().plus(key.interval());
|
||||
if (!position.isBefore(newPosition)) {
|
||||
logger.atWarning().log("Cursor has already been rolled forward.");
|
||||
return;
|
||||
}
|
||||
verify(
|
||||
position.equals(key.watermark()),
|
||||
"Partial ordering of RDE deposits broken: %s %s",
|
||||
position,
|
||||
key);
|
||||
tm().put(Cursor.create(key.cursor(), newPosition, registry));
|
||||
logger.atInfo().log(
|
||||
"Rolled forward %s on %s cursor to %s", key.cursor(), key.tld(), newPosition);
|
||||
RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,14 +18,18 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import dagger.BindsInstance;
|
||||
import dagger.Component;
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.config.CredentialModule;
|
||||
import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
@@ -33,6 +37,7 @@ import google.registry.model.host.HostResource;
|
||||
import google.registry.model.rde.RdeMode;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.Registrar.Type;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.PendingDeposit;
|
||||
@@ -50,6 +55,8 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Singleton;
|
||||
import javax.persistence.Entity;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.PipelineResult;
|
||||
@@ -58,6 +65,7 @@ import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.transforms.FlatMapElements;
|
||||
import org.apache.beam.sdk.transforms.Flatten;
|
||||
import org.apache.beam.sdk.transforms.GroupByKey;
|
||||
import org.apache.beam.sdk.transforms.Reshuffle;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
@@ -76,10 +84,15 @@ import org.joda.time.DateTime;
|
||||
* @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
|
||||
* Flex Templates</a>
|
||||
*/
|
||||
@Singleton
|
||||
public class RdePipeline implements Serializable {
|
||||
|
||||
private final RdeMarshaller marshaller;
|
||||
private final transient RdePipelineOptions options;
|
||||
private final ValidationMode mode;
|
||||
private final ImmutableSetMultimap<String, PendingDeposit> pendings;
|
||||
private final String rdeBucket;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
// Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
|
||||
// if sneaks into production we would get an extra signal.
|
||||
@@ -97,31 +110,43 @@ public class RdePipeline implements Serializable {
|
||||
+ (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
|
||||
}
|
||||
|
||||
RdePipeline(RdePipelineOptions options) throws IOException, ClassNotFoundException {
|
||||
this.marshaller = new RdeMarshaller(ValidationMode.valueOf(options.getValidationMode()));
|
||||
@Inject
|
||||
RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
|
||||
this.options = options;
|
||||
this.mode = ValidationMode.valueOf(options.getValidationMode());
|
||||
this.pendings = decodePendings(options.getPendings());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
PipelineResult run(Pipeline pipeline) {
|
||||
createFragments(pipeline);
|
||||
return pipeline.run();
|
||||
this.rdeBucket = options.getGcsBucket();
|
||||
this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
PipelineResult run() {
|
||||
return run(Pipeline.create());
|
||||
Pipeline pipeline = Pipeline.create(options);
|
||||
PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> fragments =
|
||||
createFragments(pipeline);
|
||||
persistData(fragments);
|
||||
return pipeline.run();
|
||||
}
|
||||
|
||||
PCollection<KV<PendingDeposit, DepositFragment>> createFragments(Pipeline pipeline) {
|
||||
PCollection<KV<PendingDeposit, DepositFragment>> fragments =
|
||||
PCollectionList.of(processRegistrars(pipeline))
|
||||
.and(processNonRegistrarEntities(pipeline, DomainBase.class))
|
||||
.and(processNonRegistrarEntities(pipeline, ContactResource.class))
|
||||
.and(processNonRegistrarEntities(pipeline, HostResource.class))
|
||||
.apply(Flatten.pCollections())
|
||||
.setCoder(
|
||||
KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)));
|
||||
return fragments;
|
||||
PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
|
||||
return PCollectionList.of(processRegistrars(pipeline))
|
||||
.and(processNonRegistrarEntities(pipeline, DomainBase.class))
|
||||
.and(processNonRegistrarEntities(pipeline, ContactResource.class))
|
||||
.and(processNonRegistrarEntities(pipeline, HostResource.class))
|
||||
.apply(Flatten.pCollections())
|
||||
.setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
|
||||
.apply("Group by PendingDeposit", GroupByKey.create());
|
||||
}
|
||||
|
||||
void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
|
||||
input.apply(
|
||||
"Write to GCS and update cursors",
|
||||
RdeIO.Write.builder()
|
||||
.setRdeBucket(rdeBucket)
|
||||
.setGcsUtils(gcsUtils)
|
||||
.setValidationMode(mode)
|
||||
.setStagingKeyBytes(stagingKeyBytes)
|
||||
.build());
|
||||
}
|
||||
|
||||
PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
|
||||
@@ -144,7 +169,8 @@ public class RdePipeline implements Serializable {
|
||||
.via(
|
||||
(VKey<Registrar> key) -> {
|
||||
Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
|
||||
DepositFragment fragment = marshaller.marshalRegistrar(registrar);
|
||||
DepositFragment fragment =
|
||||
new RdeMarshaller(mode).marshalRegistrar(registrar);
|
||||
return pendings.values().stream()
|
||||
.map(pending -> KV.of(pending, fragment))
|
||||
.collect(toImmutableSet());
|
||||
@@ -205,7 +231,8 @@ public class RdePipeline implements Serializable {
|
||||
Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
|
||||
// Convert resource to an XML fragment for each watermark/mode pair lazily and cache
|
||||
// the result.
|
||||
RdeFragmenter fragmenter = new RdeFragmenter(resourceAtTimes, marshaller);
|
||||
RdeFragmenter fragmenter =
|
||||
new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
|
||||
List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
|
||||
for (String tld : tlds) {
|
||||
for (PendingDeposit pending : pendings.get(tld)) {
|
||||
@@ -228,13 +255,14 @@ public class RdePipeline implements Serializable {
|
||||
* the original TLD to pending deposit map.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending)
|
||||
throws IOException, ClassNotFoundException {
|
||||
static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
|
||||
try (ObjectInputStream ois =
|
||||
new ObjectInputStream(
|
||||
new ByteArrayInputStream(
|
||||
BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
|
||||
return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
|
||||
} catch (IOException | ClassNotFoundException e) {
|
||||
throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,7 +270,7 @@ public class RdePipeline implements Serializable {
|
||||
* Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
|
||||
* worker by the pipeline launcher as a pipeline option.
|
||||
*/
|
||||
public static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
|
||||
static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
|
||||
throws IOException {
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
ObjectOutputStream oos = new ObjectOutputStream(baos);
|
||||
@@ -255,6 +283,21 @@ public class RdePipeline implements Serializable {
|
||||
public static void main(String[] args) throws IOException, ClassNotFoundException {
|
||||
PipelineOptionsFactory.register(RdePipelineOptions.class);
|
||||
RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
|
||||
new RdePipeline(options).run();
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
|
||||
}
|
||||
|
||||
@Singleton
|
||||
@Component(modules = {CredentialModule.class, ConfigModule.class})
|
||||
interface RdePipelineComponent {
|
||||
RdePipeline rdePipeline();
|
||||
|
||||
@Component.Builder
|
||||
interface Builder {
|
||||
@BindsInstance
|
||||
Builder options(RdePipelineOptions options);
|
||||
|
||||
RdePipelineComponent build();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,4 +29,14 @@ public interface RdePipelineOptions extends RegistryPipelineOptions {
|
||||
String getValidationMode();
|
||||
|
||||
void setValidationMode(String value);
|
||||
|
||||
@Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
|
||||
String getGcsBucket();
|
||||
|
||||
void setGcsBucket(String value);
|
||||
|
||||
@Description("The Base64-encoded PGP public key to encrypt the deposits.")
|
||||
String getStagingKey();
|
||||
|
||||
void setStagingKey(String value);
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import google.registry.util.UtilsModule;
|
||||
@@ -98,6 +99,7 @@ public class Spec11Pipeline implements Serializable {
|
||||
}
|
||||
|
||||
void setupPipeline(Pipeline pipeline) {
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
PCollection<Subdomain> domains =
|
||||
options.getDatabase().equals("DATASTORE")
|
||||
? readFromBigQuery(options, pipeline)
|
||||
|
||||
@@ -416,17 +416,6 @@ public final class RegistryConfig {
|
||||
return Optional.ofNullable(config.cloudDns.servicePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns size of Google Cloud Storage client connection buffer in bytes.
|
||||
*
|
||||
* @see google.registry.gcs.GcsUtils
|
||||
*/
|
||||
@Provides
|
||||
@Config("gcsBufferSize")
|
||||
public static int provideGcsBufferSize() {
|
||||
return 1024 * 1024;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the email address of the admin account on the G Suite app used to perform
|
||||
* administrative actions.
|
||||
@@ -1278,16 +1267,34 @@ public final class RegistryConfig {
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningDays")
|
||||
public static int provideDaysToExpiration(RegistryConfigSettings config) {
|
||||
public static int provideExpirationWarningDays(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningDays;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningIntervalDays")
|
||||
public static int provideExpirationWarningIntervalDays(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningIntervalDays;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("minimumRsaKeyLength")
|
||||
public static int provideMinimumRsaKeyLength(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.minimumRsaKeyLength;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningEmailBodyText")
|
||||
public static String provideExpirationWarningEmailBodyText(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningEmailBodyText;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningEmailSubjectText")
|
||||
public static String provideExpirationWarningEmailSubjectText(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningEmailSubjectText;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("allowedEcdsaCurves")
|
||||
public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
|
||||
@@ -1487,21 +1494,6 @@ public final class RegistryConfig {
|
||||
return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether to replicate cloud SQL transactions to datastore.
|
||||
*
|
||||
* <p>If true, all cloud SQL transactions will be persisted as TransactionEntity objects in the
|
||||
* Transaction table and replayed against datastore in a cron job.
|
||||
*/
|
||||
public static boolean getCloudSqlReplicateTransactions() {
|
||||
return CONFIG_SETTINGS.get().cloudSql.replicateTransactions;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void overrideCloudSqlReplicateTransactions(boolean replicateTransactions) {
|
||||
CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
|
||||
}
|
||||
|
||||
/** Returns the roid suffix to be used for the roids of all contacts and hosts. */
|
||||
public static String getContactAndHostRoidSuffix() {
|
||||
return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;
|
||||
|
||||
@@ -126,7 +126,6 @@ public class RegistryConfigSettings {
|
||||
// TODO(05012021): remove username field after it is removed from all yaml files.
|
||||
public String username;
|
||||
public String instanceConnectionName;
|
||||
public boolean replicateTransactions;
|
||||
}
|
||||
|
||||
/** Configuration for Apache Beam (Cloud Dataflow). */
|
||||
@@ -228,7 +227,10 @@ public class RegistryConfigSettings {
|
||||
public static class SslCertificateValidation {
|
||||
public Map<String, Integer> maxValidityDaysSchedule;
|
||||
public int expirationWarningDays;
|
||||
public int expirationWarningIntervalDays;
|
||||
public int minimumRsaKeyLength;
|
||||
public Set<String> allowedEcdsaCurves;
|
||||
public String expirationWarningEmailBodyText;
|
||||
public String expirationWarningEmailSubjectText;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,9 +227,6 @@ cloudSql:
|
||||
jdbcUrl: jdbc:postgresql://localhost
|
||||
# This name is used by Cloud SQL when connecting to the database.
|
||||
instanceConnectionName: project-id:region:instance-id
|
||||
# Set this to true to replicate cloud SQL transactions to datastore in the
|
||||
# background.
|
||||
replicateTransactions: false
|
||||
|
||||
cloudDns:
|
||||
# Set both properties to null in Production.
|
||||
@@ -452,6 +449,15 @@ sslCertificateValidation:
|
||||
# The number of days before a certificate expires that indicates the
|
||||
# certificate is nearing expiration and warnings should be sent.
|
||||
expirationWarningDays: 30
|
||||
# The minimum number of days between two successive expiring notification emails.
|
||||
expirationWarningIntervalDays: 15
|
||||
# Text for expiring certificate notification email subject.
|
||||
expirationWarningEmailSubjectText: Certificate Expring Within 30 Days.
|
||||
# Text for expiring certificate notification email body that accepts 3 parameters:
|
||||
# registrar name, certificate type, and expiration date, respectively.
|
||||
expirationWarningEmailBodyText: |
|
||||
Hello Registrar %s,
|
||||
The %s certificate is expiring on %s.
|
||||
# The minimum number of bits an RSA key must contain.
|
||||
minimumRsaKeyLength: 2048
|
||||
# The ECDSA curves that are allowed for public keys.
|
||||
|
||||
@@ -284,9 +284,10 @@ public class CloudDnsWriter extends BaseDnsWriter {
|
||||
.filter(hostName -> hostName.endsWith("." + domainName) && !hostName.equals(domainName));
|
||||
}
|
||||
|
||||
/** Mutate the zone with the provided {@code desiredRecords}. */
|
||||
/** Mutate the zone with the provided map of hostnames to desired DNS records. */
|
||||
@VisibleForTesting
|
||||
void mutateZone(ImmutableMap<String, ImmutableSet<ResourceRecordSet>> desiredRecords) {
|
||||
logger.atInfo().log("Updating DNS records for hostname(s) %s.", desiredRecords.keySet());
|
||||
// Fetch all existing records for names that this writer is trying to modify
|
||||
ImmutableSet.Builder<ResourceRecordSet> flattenedExistingRecords = new ImmutableSet.Builder<>();
|
||||
|
||||
@@ -313,7 +314,12 @@ public class CloudDnsWriter extends BaseDnsWriter {
|
||||
desiredRecords.values().forEach(flattenedDesiredRecords::addAll);
|
||||
|
||||
// Delete all existing records and add back the desired records
|
||||
updateResourceRecords(flattenedDesiredRecords.build(), flattenedExistingRecords.build());
|
||||
try {
|
||||
updateResourceRecords(flattenedDesiredRecords.build(), flattenedExistingRecords.build());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(
|
||||
"Error updating DNS records for hostname(s) " + desiredRecords.keySet(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -363,10 +369,12 @@ public class CloudDnsWriter extends BaseDnsWriter {
|
||||
*
|
||||
* @throws ZoneStateException if the operation could not be completely successfully because the
|
||||
* records to delete do not exist, already exist or have been modified with different
|
||||
* attributes since being queried.
|
||||
* attributes since being queried. These errors will be retried.
|
||||
* @throws IOException on non-retryable API errors, e.g. invalid request.
|
||||
*/
|
||||
private void updateResourceRecords(
|
||||
ImmutableSet<ResourceRecordSet> additions, ImmutableSet<ResourceRecordSet> deletions) {
|
||||
ImmutableSet<ResourceRecordSet> additions, ImmutableSet<ResourceRecordSet> deletions)
|
||||
throws IOException {
|
||||
// Find records that are both in additions and deletions, so we can remove them from both before
|
||||
// requesting the change. This is mostly for optimization reasons - not doing so doesn't affect
|
||||
// the result.
|
||||
@@ -392,17 +400,15 @@ public class CloudDnsWriter extends BaseDnsWriter {
|
||||
GoogleJsonError err = e.getDetails();
|
||||
// We did something really wrong here, just give up and re-throw
|
||||
if (err == null || err.getErrors().size() > 1) {
|
||||
throw new RuntimeException(e);
|
||||
throw e;
|
||||
}
|
||||
String errorReason = err.getErrors().get(0).getReason();
|
||||
|
||||
if (RETRYABLE_EXCEPTION_REASONS.contains(errorReason)) {
|
||||
throw new ZoneStateException(errorReason);
|
||||
} else {
|
||||
throw new RuntimeException(e);
|
||||
throw e;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.base.Verify.verifyNotNull;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
@@ -24,11 +23,10 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Suppliers;
|
||||
@@ -79,9 +77,9 @@ public class ExportDomainListsAction implements Runnable {
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject DriveConnection driveConnection;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("domainListsGcsBucket") String gcsBucket;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject ExportDomainListsAction() {}
|
||||
|
||||
@Override
|
||||
@@ -95,7 +93,7 @@ public class ExportDomainListsAction implements Runnable {
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(clock.nowUtc(), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsUtils),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
@@ -134,7 +132,7 @@ public class ExportDomainListsAction implements Runnable {
|
||||
String domainsList = Joiner.on("\n").join(domains);
|
||||
logger.atInfo().log(
|
||||
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsUtils);
|
||||
exportToDrive(tld, domainsList, driveConnection);
|
||||
});
|
||||
}
|
||||
@@ -168,11 +166,9 @@ public class ExportDomainListsAction implements Runnable {
|
||||
}
|
||||
|
||||
protected static boolean exportToGcs(
|
||||
String tld, String domains, String gcsBucket, int gcsBufferSize) {
|
||||
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
String tld, String domains, String gcsBucket, GcsUtils gcsUtils) {
|
||||
BlobId blobId = BlobId.of(gcsBucket, tld + ".txt");
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (Throwable e) {
|
||||
@@ -214,7 +210,7 @@ public class ExportDomainListsAction implements Runnable {
|
||||
Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection());
|
||||
|
||||
private final String gcsBucket;
|
||||
private final int gcsBufferSize;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
/**
|
||||
* Non-serializable {@link DriveConnection} that will be created when an instance of {@link
|
||||
@@ -224,9 +220,9 @@ public class ExportDomainListsAction implements Runnable {
|
||||
*/
|
||||
private transient DriveConnection driveConnection;
|
||||
|
||||
public ExportDomainListsReducer(String gcsBucket, int gcsBufferSize) {
|
||||
public ExportDomainListsReducer(String gcsBucket, GcsUtils gcsUtils) {
|
||||
this.gcsBucket = gcsBucket;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@@ -240,7 +236,7 @@ public class ExportDomainListsAction implements Runnable {
|
||||
ImmutableList<String> domains = ImmutableList.sortedCopyOf(() -> fqdns);
|
||||
String domainsList = Joiner.on('\n').join(domains);
|
||||
logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize)) {
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsUtils)) {
|
||||
getContext().incrementCounter("domain lists successful written out to GCS");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to GCS");
|
||||
|
||||
@@ -113,7 +113,7 @@ public class ExportPremiumTermsAction implements Runnable {
|
||||
"Skipping premium terms export for TLD %s because Drive folder isn't specified", tld);
|
||||
return Optional.of("Skipping export because no Drive folder is associated with this TLD");
|
||||
}
|
||||
if (!registry.getPremiumList().isPresent()) {
|
||||
if (!registry.getPremiumListName().isPresent()) {
|
||||
logger.atInfo().log("No premium terms to export for TLD %s", tld);
|
||||
return Optional.of("No premium lists configured");
|
||||
}
|
||||
@@ -137,8 +137,8 @@ public class ExportPremiumTermsAction implements Runnable {
|
||||
}
|
||||
|
||||
private String getFormattedPremiumTerms(Registry registry) {
|
||||
checkState(registry.getPremiumList().isPresent(), "%s does not have a premium list", tld);
|
||||
String premiumListName = registry.getPremiumList().get().getName();
|
||||
checkState(registry.getPremiumListName().isPresent(), "%s does not have a premium list", tld);
|
||||
String premiumListName = registry.getPremiumListName().get();
|
||||
checkState(
|
||||
PremiumListDao.getLatestRevision(premiumListName).isPresent(),
|
||||
"Could not load premium list for " + tld);
|
||||
|
||||
@@ -63,7 +63,7 @@ public class ExportReservedTermsAction implements Runnable {
|
||||
try {
|
||||
Registry registry = Registry.get(tld);
|
||||
String resultMsg;
|
||||
if (registry.getReservedLists().isEmpty() && isNullOrEmpty(registry.getDriveFolderId())) {
|
||||
if (registry.getReservedListNames().isEmpty() && isNullOrEmpty(registry.getDriveFolderId())) {
|
||||
resultMsg = "No reserved lists configured";
|
||||
logger.atInfo().log("No reserved terms to export for TLD %s", tld);
|
||||
} else if (registry.getDriveFolderId() == null) {
|
||||
|
||||
@@ -15,11 +15,11 @@
|
||||
package google.registry.export;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.registry.label.ReservedList.ReservedListEntry;
|
||||
import google.registry.model.registry.label.ReservedListDao;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import javax.inject.Inject;
|
||||
@@ -39,8 +39,13 @@ public final class ExportUtils {
|
||||
public String exportReservedTerms(Registry registry) {
|
||||
StringBuilder termsBuilder = new StringBuilder(reservedTermsExportDisclaimer).append("\n");
|
||||
Set<String> reservedTerms = new TreeSet<>();
|
||||
for (Key<ReservedList> key : registry.getReservedLists()) {
|
||||
ReservedList reservedList = ReservedList.load(key).get();
|
||||
for (String reservedListName : registry.getReservedListNames()) {
|
||||
ReservedList reservedList =
|
||||
ReservedListDao.getLatestRevision(reservedListName)
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalStateException(
|
||||
String.format("Reserved list %s does not exist", reservedListName)));
|
||||
if (reservedList.getShouldPublish()) {
|
||||
for (ReservedListEntry entry : reservedList.getReservedListEntries().values()) {
|
||||
reservedTerms.add(entry.getLabel());
|
||||
|
||||
@@ -24,6 +24,7 @@ import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.DateTimeUtils;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.StringWriter;
|
||||
import java.security.PublicKey;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.security.cert.CertificateFactory;
|
||||
@@ -32,10 +33,15 @@ import java.security.interfaces.ECPublicKey;
|
||||
import java.security.interfaces.RSAPublicKey;
|
||||
import java.util.Date;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import org.bouncycastle.jcajce.provider.asymmetric.util.EC5Util;
|
||||
import org.bouncycastle.jce.ECNamedCurveTable;
|
||||
import org.bouncycastle.jce.spec.ECNamedCurveParameterSpec;
|
||||
import org.bouncycastle.jce.spec.ECParameterSpec;
|
||||
import org.bouncycastle.openssl.jcajce.JcaMiscPEMGenerator;
|
||||
import org.bouncycastle.util.io.pem.PemObjectGenerator;
|
||||
import org.bouncycastle.util.io.pem.PemWriter;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Days;
|
||||
|
||||
@@ -43,10 +49,11 @@ import org.joda.time.Days;
|
||||
public class CertificateChecker {
|
||||
|
||||
private final ImmutableSortedMap<DateTime, Integer> maxValidityLengthSchedule;
|
||||
private final int daysToExpiration;
|
||||
private final int expirationWarningDays;
|
||||
private final int minimumRsaKeyLength;
|
||||
private final Clock clock;
|
||||
private final ImmutableSet<String> allowedEcdsaCurves;
|
||||
private final int expirationWarningIntervalDays;
|
||||
|
||||
/**
|
||||
* Constructs a CertificateChecker instance with the specified configuration parameters.
|
||||
@@ -72,6 +79,7 @@ public class CertificateChecker {
|
||||
@Config("maxValidityDaysSchedule")
|
||||
ImmutableSortedMap<DateTime, Integer> maxValidityDaysSchedule,
|
||||
@Config("expirationWarningDays") int expirationWarningDays,
|
||||
@Config("expirationWarningIntervalDays") int expirationWarningIntervalDays,
|
||||
@Config("minimumRsaKeyLength") int minimumRsaKeyLength,
|
||||
@Config("allowedEcdsaCurves") ImmutableSet<String> allowedEcdsaCurves,
|
||||
Clock clock) {
|
||||
@@ -79,12 +87,46 @@ public class CertificateChecker {
|
||||
maxValidityDaysSchedule.containsKey(START_OF_TIME),
|
||||
"Max validity length schedule must contain an entry for START_OF_TIME");
|
||||
this.maxValidityLengthSchedule = maxValidityDaysSchedule;
|
||||
this.daysToExpiration = expirationWarningDays;
|
||||
this.expirationWarningDays = expirationWarningDays;
|
||||
this.minimumRsaKeyLength = minimumRsaKeyLength;
|
||||
this.allowedEcdsaCurves = allowedEcdsaCurves;
|
||||
this.expirationWarningIntervalDays = expirationWarningIntervalDays;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
private static int getValidityLengthInDays(X509Certificate certificate) {
|
||||
DateTime start = DateTime.parse(certificate.getNotBefore().toInstant().toString());
|
||||
DateTime end = DateTime.parse(certificate.getNotAfter().toInstant().toString());
|
||||
return Days.daysBetween(start.withTimeAtStartOfDay(), end.withTimeAtStartOfDay()).getDays();
|
||||
}
|
||||
|
||||
/** Checks if the curve used for a public key is in the list of acceptable curves. */
|
||||
private static boolean checkCurveName(PublicKey key, ImmutableSet<String> allowedEcdsaCurves) {
|
||||
ECParameterSpec params;
|
||||
// These 2 different instances of PublicKey need to be handled separately since their OIDs are
|
||||
// encoded differently. More details on this can be found at
|
||||
// https://stackoverflow.com/questions/49895713/how-to-find-the-matching-curve-name-from-an-ecpublickey.
|
||||
if (key instanceof ECPublicKey) {
|
||||
ECPublicKey ecKey = (ECPublicKey) key;
|
||||
params = EC5Util.convertSpec(ecKey.getParams(), false);
|
||||
} else if (key instanceof org.bouncycastle.jce.interfaces.ECPublicKey) {
|
||||
org.bouncycastle.jce.interfaces.ECPublicKey ecKey =
|
||||
(org.bouncycastle.jce.interfaces.ECPublicKey) key;
|
||||
params = ecKey.getParameters();
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unrecognized instance of PublicKey.");
|
||||
}
|
||||
return allowedEcdsaCurves.stream()
|
||||
.anyMatch(
|
||||
curve -> {
|
||||
ECNamedCurveParameterSpec cParams = ECNamedCurveTable.getParameterSpec(curve);
|
||||
return cParams.getN().equals(params.getN())
|
||||
&& cParams.getH().equals(params.getH())
|
||||
&& cParams.getCurve().equals(params.getCurve())
|
||||
&& cParams.getG().equals(params.getG());
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the given certificate string for violations and throws an exception if any violations
|
||||
* exist.
|
||||
@@ -156,67 +198,57 @@ public class CertificateChecker {
|
||||
* the violations the certificate has.
|
||||
*/
|
||||
public ImmutableSet<CertificateViolation> checkCertificate(String certificateString) {
|
||||
X509Certificate certificate;
|
||||
return checkCertificate(getCertificate(certificateString));
|
||||
}
|
||||
|
||||
/** Converts the given string to a certificate object. */
|
||||
public X509Certificate getCertificate(String certificateStr) {
|
||||
X509Certificate certificate;
|
||||
try {
|
||||
certificate =
|
||||
(X509Certificate)
|
||||
CertificateFactory.getInstance("X509")
|
||||
.generateCertificate(new ByteArrayInputStream(certificateString.getBytes(UTF_8)));
|
||||
.generateCertificate(new ByteArrayInputStream(certificateStr.getBytes(UTF_8)));
|
||||
} catch (CertificateException e) {
|
||||
throw new IllegalArgumentException("Unable to read given certificate.");
|
||||
throw new IllegalArgumentException(
|
||||
String.format("Unable to read given certificate %s", certificateStr), e);
|
||||
}
|
||||
|
||||
return checkCertificate(certificate);
|
||||
return certificate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether the certificate is nearing expiration.
|
||||
*
|
||||
* <p>Note that this is <i>all</i> that it checks. The certificate itself may well be expired or
|
||||
* not yet valid and this message will still return false. So you definitely want to pair a call
|
||||
* to this method with a call to {@link #checkCertificate} to determine other issues with the
|
||||
* certificate that may be occurring.
|
||||
*/
|
||||
public boolean isNearingExpiration(X509Certificate certificate) {
|
||||
Date nearingExpirationDate =
|
||||
DateTime.parse(certificate.getNotAfter().toInstant().toString())
|
||||
.minusDays(daysToExpiration)
|
||||
.toDate();
|
||||
return clock.nowUtc().toDate().after(nearingExpirationDate);
|
||||
}
|
||||
|
||||
private static int getValidityLengthInDays(X509Certificate certificate) {
|
||||
DateTime start = DateTime.parse(certificate.getNotBefore().toInstant().toString());
|
||||
DateTime end = DateTime.parse(certificate.getNotAfter().toInstant().toString());
|
||||
return Days.daysBetween(start.withTimeAtStartOfDay(), end.withTimeAtStartOfDay()).getDays();
|
||||
}
|
||||
|
||||
/** Checks if the curve used for a public key is in the list of acceptable curves. */
|
||||
private static boolean checkCurveName(PublicKey key, ImmutableSet<String> allowedEcdsaCurves) {
|
||||
org.bouncycastle.jce.spec.ECParameterSpec params;
|
||||
// These 2 different instances of PublicKey need to be handled separately since their OIDs are
|
||||
// encoded differently. More details on this can be found at
|
||||
// https://stackoverflow.com/questions/49895713/how-to-find-the-matching-curve-name-from-an-ecpublickey.
|
||||
if (key instanceof ECPublicKey) {
|
||||
ECPublicKey ecKey = (ECPublicKey) key;
|
||||
params = EC5Util.convertSpec(ecKey.getParams(), false);
|
||||
} else if (key instanceof org.bouncycastle.jce.interfaces.ECPublicKey) {
|
||||
org.bouncycastle.jce.interfaces.ECPublicKey ecKey =
|
||||
(org.bouncycastle.jce.interfaces.ECPublicKey) key;
|
||||
params = ecKey.getParameters();
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unrecognized instance of PublicKey.");
|
||||
/** Serializes the certificate object to a certificate string. */
|
||||
public String serializeCertificate(X509Certificate certificate) throws Exception {
|
||||
StringWriter sw = new StringWriter();
|
||||
try (PemWriter pw = new PemWriter(sw)) {
|
||||
PemObjectGenerator generator = new JcaMiscPEMGenerator(certificate);
|
||||
pw.writeObject(generator);
|
||||
}
|
||||
return allowedEcdsaCurves.stream()
|
||||
.anyMatch(
|
||||
curve -> {
|
||||
ECNamedCurveParameterSpec cParams = ECNamedCurveTable.getParameterSpec(curve);
|
||||
return cParams.getN().equals(params.getN())
|
||||
&& cParams.getH().equals(params.getH())
|
||||
&& cParams.getCurve().equals(params.getCurve())
|
||||
&& cParams.getG().equals(params.getG());
|
||||
});
|
||||
return sw.toString();
|
||||
}
|
||||
|
||||
/** Returns whether the client should receive a notification email. */
|
||||
public boolean shouldReceiveExpiringNotification(
|
||||
@Nullable DateTime lastExpiringNotificationSentDate, String certificateStr) {
|
||||
X509Certificate certificate = getCertificate(certificateStr);
|
||||
DateTime now = clock.nowUtc();
|
||||
// expiration date is one day after lastValidDate
|
||||
Date lastValidDate = certificate.getNotAfter();
|
||||
if (lastValidDate.before(now.toDate())) {
|
||||
return false;
|
||||
}
|
||||
/*
|
||||
* Client should receive a notification if :
|
||||
* 1) client has never received notification and the certificate has entered
|
||||
* the expiring period, OR
|
||||
* 2) client has received notification but the interval between now and
|
||||
* lastExpiringNotificationSentDate is greater than expirationWarningIntervalDays.
|
||||
*/
|
||||
return !lastValidDate.after(now.plusDays(expirationWarningDays).toDate())
|
||||
&& (lastExpiringNotificationSentDate == null
|
||||
|| !lastExpiringNotificationSentDate
|
||||
.plusDays(expirationWarningIntervalDays)
|
||||
.toDate()
|
||||
.after(now.toDate()));
|
||||
}
|
||||
|
||||
private String getViolationDisplayMessage(CertificateViolation certificateViolation) {
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.gcs;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
|
||||
/** Dagger module for {@link GcsService}. */
|
||||
@Module
|
||||
public final class GcsServiceModule {
|
||||
|
||||
private static final GcsService gcsService =
|
||||
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
|
||||
|
||||
@Provides
|
||||
static GcsService provideGcsService() {
|
||||
return gcsService;
|
||||
}
|
||||
}
|
||||
@@ -14,30 +14,39 @@
|
||||
|
||||
package google.registry.gcs;
|
||||
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterables.getLast;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.cloud.storage.Blob;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.Storage;
|
||||
import com.google.cloud.storage.Storage.BlobListOption;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.CredentialModule.DefaultCredential;
|
||||
import google.registry.util.GoogleCredentialsBundle;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.io.Serializable;
|
||||
import java.nio.channels.Channels;
|
||||
import javax.annotation.CheckReturnValue;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Utilities for working with Google Cloud Storage. */
|
||||
public class GcsUtils {
|
||||
/**
|
||||
* Utilities for working with Google Cloud Storage.
|
||||
*
|
||||
* <p>It is {@link Serializable} so that it can be used in MapReduce or Beam.
|
||||
*/
|
||||
public class GcsUtils implements Serializable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@@ -48,30 +57,65 @@ public class GcsUtils {
|
||||
.put("json", MediaType.JSON_UTF_8)
|
||||
.build();
|
||||
|
||||
private final GcsService gcsService;
|
||||
private final int bufferSize;
|
||||
private final StorageOptions storageOptions;
|
||||
|
||||
@Inject
|
||||
public GcsUtils(GcsService gcsService, @Config("gcsBufferSize") int bufferSize) {
|
||||
this.gcsService = gcsService;
|
||||
this.bufferSize = bufferSize;
|
||||
private Storage storage() {
|
||||
return storageOptions.getService();
|
||||
}
|
||||
|
||||
/** Opens a GCS file for reading as an {@link InputStream} with prefetching. */
|
||||
@Inject
|
||||
public GcsUtils(@DefaultCredential GoogleCredentialsBundle credentialsBundle) {
|
||||
this(
|
||||
StorageOptions.newBuilder()
|
||||
.setCredentials(credentialsBundle.getGoogleCredentials())
|
||||
.build());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public GcsUtils(StorageOptions storageOptions) {
|
||||
this.storageOptions = storageOptions;
|
||||
}
|
||||
|
||||
/** Opens a GCS file for reading as an {@link InputStream}. */
|
||||
@CheckReturnValue
|
||||
public InputStream openInputStream(GcsFilename filename) {
|
||||
return Channels.newInputStream(gcsService.openPrefetchingReadChannel(filename, 0, bufferSize));
|
||||
public InputStream openInputStream(BlobId blobId) {
|
||||
return Channels.newInputStream(storage().reader(blobId));
|
||||
}
|
||||
|
||||
/** Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files. */
|
||||
@CheckReturnValue
|
||||
public OutputStream openOutputStream(GcsFilename filename) throws IOException {
|
||||
return Channels.newOutputStream(gcsService.createOrReplace(filename, getOptions(filename)));
|
||||
public OutputStream openOutputStream(BlobId blobId) {
|
||||
return Channels.newOutputStream(storage().writer(createBlobInfo(blobId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files and setting
|
||||
* the given metadata.
|
||||
*/
|
||||
@CheckReturnValue
|
||||
public OutputStream openOutputStream(BlobId blobId, ImmutableMap<String, String> metadata) {
|
||||
return Channels.newOutputStream(
|
||||
storage().writer(BlobInfo.newBuilder(blobId).setMetadata(metadata).build()));
|
||||
}
|
||||
|
||||
/** Creates a GCS file with the given byte contents, overwriting existing files. */
|
||||
public void createFromBytes(GcsFilename filename, byte[] bytes) throws IOException {
|
||||
gcsService.createOrReplace(filename, getOptions(filename), ByteBuffer.wrap(bytes));
|
||||
public void createFromBytes(BlobId blobId, byte[] bytes) throws StorageException {
|
||||
createFromBytes(createBlobInfo(blobId), bytes);
|
||||
}
|
||||
|
||||
/** Creates a GCS file with the given byte contents and metadata, overwriting existing files. */
|
||||
public void createFromBytes(BlobInfo blobInfo, byte[] bytes) throws StorageException {
|
||||
storage().create(blobInfo, bytes);
|
||||
}
|
||||
|
||||
/** Read the content of the given GCS file and return it in a byte array. */
|
||||
public byte[] readBytesFrom(BlobId blobId) throws StorageException {
|
||||
return storage().readAllBytes(blobId);
|
||||
}
|
||||
|
||||
/** Delete the given GCS file. */
|
||||
public void delete(BlobId blobId) throws StorageException {
|
||||
storage().delete(blobId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -86,37 +130,55 @@ public class GcsUtils {
|
||||
*/
|
||||
public ImmutableList<String> listFolderObjects(String bucketName, String prefix)
|
||||
throws IOException {
|
||||
ListResult result =
|
||||
gcsService.list(bucketName, new ListOptions.Builder().setPrefix(prefix).build());
|
||||
final ImmutableList.Builder<String> builder = new ImmutableList.Builder<>();
|
||||
result.forEachRemaining(
|
||||
listItem -> {
|
||||
if (!listItem.isDirectory()) {
|
||||
builder.add(listItem.getName().replaceFirst(prefix, ""));
|
||||
}
|
||||
});
|
||||
return builder.build();
|
||||
return Streams.stream(storage().list(bucketName, BlobListOption.prefix(prefix)).iterateAll())
|
||||
.map(blob -> blob.getName().substring(prefix.length()))
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
|
||||
/** Returns {@code true} if a file exists and is non-empty on Google Cloud Storage. */
|
||||
public boolean existsAndNotEmpty(GcsFilename file) {
|
||||
GcsFileMetadata metadata;
|
||||
public boolean existsAndNotEmpty(BlobId blobId) {
|
||||
try {
|
||||
metadata = gcsService.getMetadata(file);
|
||||
} catch (IOException e) {
|
||||
Blob blob = storage().get(blobId);
|
||||
return blob != null && blob.getSize() > 0;
|
||||
} catch (StorageException e) {
|
||||
logger.atWarning().withCause(e).log("Failed to check if GCS file exists");
|
||||
return false;
|
||||
}
|
||||
return metadata != null && metadata.getLength() > 0;
|
||||
}
|
||||
|
||||
/** Determines most appropriate {@link GcsFileOptions} based on filename extension. */
|
||||
private static GcsFileOptions getOptions(GcsFilename filename) {
|
||||
GcsFileOptions.Builder builder = new GcsFileOptions.Builder().cacheControl("no-cache");
|
||||
MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(filename.getObjectName())));
|
||||
/** Returns the user defined metadata of a GCS file if the file exists, or an empty map. */
|
||||
public ImmutableMap<String, String> getMetadata(BlobId blobId) throws StorageException {
|
||||
Blob blob = storage().get(blobId);
|
||||
return blob == null ? ImmutableMap.of() : ImmutableMap.copyOf(blob.getMetadata());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the {@link BlobInfo} of the given GCS file.
|
||||
*
|
||||
* <p>Note that a {@link Blob} is returned, but on the {@link BlobInfo} part of it is usable.
|
||||
*/
|
||||
public BlobInfo getBlobInfo(BlobId blobId) throws StorageException {
|
||||
return storage().get(blobId);
|
||||
}
|
||||
|
||||
/** Determines most appropriate {@link BlobInfo} based on filename extension. */
|
||||
private static BlobInfo createBlobInfo(BlobId blobId) {
|
||||
BlobInfo.Builder builder = BlobInfo.newBuilder(blobId).setCacheControl("no-cache");
|
||||
MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(blobId.getName())));
|
||||
if (mediaType != null) {
|
||||
builder = builder.mimeType(mediaType.type());
|
||||
builder = builder.setContentType(mediaType.toString());
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// These two methods are needed to check whether serialization is done correctly in tests.
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
return obj instanceof GcsUtils && ((GcsUtils) obj).storageOptions.equals(storageOptions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return storageOptions.hashCode();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,14 +28,8 @@ import org.bouncycastle.openpgp.PGPKeyPair;
|
||||
import org.bouncycastle.openpgp.PGPPrivateKey;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
|
||||
/**
|
||||
* A {@link Keyring} implementation which stores encrypted secrets in Datastore and decrypts them
|
||||
* using encryption keys stored in Cloud KMS.
|
||||
*
|
||||
* @see <a href="https://cloud.google.com/kms/docs/">Google Cloud Key Management Service
|
||||
* Documentation</a>
|
||||
*/
|
||||
// TODO(2021-07-01): rename this class to SecretManagerKeyring and delete KmsSecretRevision
|
||||
/** A {@link Keyring} implementation which stores sensitive data in the Secret Manager. */
|
||||
// TODO(2021-08-01): rename this class to SecretManagerKeyring and update config files.
|
||||
public class KmsKeyring implements Keyring {
|
||||
|
||||
/** Key labels for private key secrets. */
|
||||
|
||||
@@ -43,8 +43,6 @@ import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.label.PremiumList;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.server.KmsSecretRevision;
|
||||
import google.registry.model.server.Lock;
|
||||
import google.registry.model.server.ServerSecret;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
@@ -88,8 +86,6 @@ public final class EntityClasses {
|
||||
HistoryEntry.class,
|
||||
HostHistory.class,
|
||||
HostResource.class,
|
||||
KmsSecret.class,
|
||||
KmsSecretRevision.class,
|
||||
LastSqlTransaction.class,
|
||||
Lock.class,
|
||||
PollMessage.class,
|
||||
|
||||
@@ -225,7 +225,7 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton
|
||||
@VisibleForTesting
|
||||
static TimedTransitionProperty<MigrationState, MigrationStateTransition> getUncached() {
|
||||
return ofyTm()
|
||||
.transact(
|
||||
.transactNew(
|
||||
() ->
|
||||
ofyTm()
|
||||
.loadSingleton(DatabaseMigrationStateSchedule.class)
|
||||
|
||||
@@ -136,9 +136,11 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
|
||||
}
|
||||
|
||||
// Used to fill out the contactBase field during asynchronous replay
|
||||
public static void beforeSqlSave(ContactHistory contactHistory) {
|
||||
contactHistory.contactBase =
|
||||
jpaTm().loadByKey(VKey.createSql(ContactResource.class, contactHistory.getContactRepoId()));
|
||||
@Override
|
||||
public void beforeSqlSaveOnReplay() {
|
||||
if (contactBase == null) {
|
||||
contactBase = jpaTm().getEntityManager().find(ContactResource.class, getContactRepoId());
|
||||
}
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link ContactHistory} entity. */
|
||||
|
||||
@@ -294,9 +294,26 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
|
||||
}
|
||||
|
||||
// Used to fill out the domainContent field during asynchronous replay
|
||||
public static void beforeSqlSave(DomainHistory domainHistory) {
|
||||
domainHistory.domainContent =
|
||||
jpaTm().loadByKey(VKey.createSql(DomainBase.class, domainHistory.getDomainRepoId()));
|
||||
@Override
|
||||
public void beforeSqlSaveOnReplay() {
|
||||
if (domainContent == null) {
|
||||
domainContent = jpaTm().getEntityManager().find(DomainBase.class, getDomainRepoId());
|
||||
fillAuxiliaryFieldsFromDomain(this);
|
||||
}
|
||||
}
|
||||
|
||||
private static void fillAuxiliaryFieldsFromDomain(DomainHistory domainHistory) {
|
||||
if (domainHistory.domainContent != null) {
|
||||
domainHistory.nsHosts = nullToEmptyImmutableCopy(domainHistory.domainContent.nsHosts);
|
||||
domainHistory.dsDataHistories =
|
||||
nullToEmptyImmutableCopy(domainHistory.domainContent.getDsData()).stream()
|
||||
.map(dsData -> DomainDsDataHistory.createFrom(domainHistory.id, dsData))
|
||||
.collect(toImmutableSet());
|
||||
domainHistory.gracePeriodHistories =
|
||||
nullToEmptyImmutableCopy(domainHistory.domainContent.getGracePeriods()).stream()
|
||||
.map(gracePeriod -> GracePeriodHistory.createFrom(domainHistory.id, gracePeriod))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link DomainHistory} entity. */
|
||||
@@ -391,17 +408,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
|
||||
// Note that we cannot assert that instance.domainContent is not null here because this
|
||||
// builder is also used to convert legacy HistoryEntry objects to DomainHistory, when
|
||||
// domainContent is not available.
|
||||
if (instance.domainContent != null) {
|
||||
instance.nsHosts = nullToEmptyImmutableCopy(instance.domainContent.nsHosts);
|
||||
instance.dsDataHistories =
|
||||
nullToEmptyImmutableCopy(instance.domainContent.getDsData()).stream()
|
||||
.map(dsData -> DomainDsDataHistory.createFrom(instance.id, dsData))
|
||||
.collect(toImmutableSet());
|
||||
instance.gracePeriodHistories =
|
||||
nullToEmptyImmutableCopy(instance.domainContent.getGracePeriods()).stream()
|
||||
.map(gracePeriod -> GracePeriodHistory.createFrom(instance.id, gracePeriod))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
fillAuxiliaryFieldsFromDomain(instance);
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,9 +136,11 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
|
||||
}
|
||||
|
||||
// Used to fill out the hostBase field during asynchronous replay
|
||||
public static void beforeSqlSave(HostHistory hostHistory) {
|
||||
hostHistory.hostBase =
|
||||
jpaTm().loadByKey(VKey.createSql(HostResource.class, hostHistory.getHostRepoId()));
|
||||
@Override
|
||||
public void beforeSqlSaveOnReplay() {
|
||||
if (hostBase == null) {
|
||||
hostBase = jpaTm().getEntityManager().find(HostResource.class, getHostRepoId());
|
||||
}
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link HostHistory} entity. */
|
||||
|
||||
+1
-3
@@ -39,9 +39,7 @@ public final class StaticPremiumListPricingEngine implements PremiumPricingEngin
|
||||
String label = InternetDomainName.from(fullyQualifiedDomainName).parts().get(0);
|
||||
Registry registry = Registry.get(checkNotNull(tld, "tld"));
|
||||
Optional<Money> premiumPrice =
|
||||
registry
|
||||
.getPremiumList()
|
||||
.flatMap(listKey -> PremiumListDao.getPremiumPrice(listKey.getName(), label));
|
||||
registry.getPremiumListName().flatMap(pl -> PremiumListDao.getPremiumPrice(pl, label));
|
||||
return DomainPrices.create(
|
||||
premiumPrice.isPresent(),
|
||||
premiumPrice.orElse(registry.getStandardCreateCost()),
|
||||
|
||||
@@ -154,7 +154,7 @@ public final class RdeRevision extends BackupGroupRoot implements NonReplicatedE
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link RdeRevision} entity. */
|
||||
static class RdeRevisionId extends ImmutableObject implements Serializable {
|
||||
public static class RdeRevisionId extends ImmutableObject implements Serializable {
|
||||
|
||||
String tld;
|
||||
|
||||
@@ -169,7 +169,7 @@ public final class RdeRevision extends BackupGroupRoot implements NonReplicatedE
|
||||
/** Hibernate requires this default constructor. */
|
||||
private RdeRevisionId() {}
|
||||
|
||||
static RdeRevisionId create(String tld, LocalDate date, RdeMode mode) {
|
||||
public static RdeRevisionId create(String tld, LocalDate date, RdeMode mode) {
|
||||
RdeRevisionId instance = new RdeRevisionId();
|
||||
instance.tld = tld;
|
||||
instance.date = date;
|
||||
|
||||
@@ -37,6 +37,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
|
||||
import static google.registry.util.CollectionUtils.nullToEmptyImmutableSortedCopy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.PasswordUtils.SALT_SUPPLIER;
|
||||
import static google.registry.util.PasswordUtils.hashPassword;
|
||||
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
@@ -458,6 +459,14 @@ public class Registrar extends ImmutableObject
|
||||
/** The time that the certificate was last updated. */
|
||||
DateTime lastCertificateUpdateTime;
|
||||
|
||||
/** The time that an expiring certificate notification email was sent to the registrar. */
|
||||
DateTime lastExpiringCertNotificationSentDate = START_OF_TIME;
|
||||
|
||||
/**
|
||||
* The time that an expiring failover certificate notification email was sent to the registrar.
|
||||
*/
|
||||
DateTime lastExpiringFailoverCertNotificationSentDate = START_OF_TIME;
|
||||
|
||||
/** Telephone support passcode (5-digit numeric) */
|
||||
String phonePasscode;
|
||||
|
||||
@@ -508,6 +517,14 @@ public class Registrar extends ImmutableObject
|
||||
return lastCertificateUpdateTime;
|
||||
}
|
||||
|
||||
public DateTime getLastExpiringCertNotificationSentDate() {
|
||||
return lastExpiringCertNotificationSentDate;
|
||||
}
|
||||
|
||||
public DateTime getLastExpiringFailoverCertNotificationSentDate() {
|
||||
return lastExpiringFailoverCertNotificationSentDate;
|
||||
}
|
||||
|
||||
public String getRegistrarName() {
|
||||
return registrarName;
|
||||
}
|
||||
@@ -671,6 +688,10 @@ public class Registrar extends ImmutableObject
|
||||
.putString("creationTime", creationTime.getTimestamp())
|
||||
.putString("lastUpdateTime", lastUpdateTime.getTimestamp())
|
||||
.putString("lastCertificateUpdateTime", lastCertificateUpdateTime)
|
||||
.putString("lastExpiringCertNotificationSentDate", lastExpiringCertNotificationSentDate)
|
||||
.putString(
|
||||
"lastExpiringFailoverCertNotificationSentDate",
|
||||
lastExpiringFailoverCertNotificationSentDate)
|
||||
.put("registrarName", registrarName)
|
||||
.put("type", type)
|
||||
.put("state", state)
|
||||
@@ -839,6 +860,19 @@ public class Registrar extends ImmutableObject
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setLastExpiringCertNotificationSentDate(DateTime now) {
|
||||
checkArgumentNotNull(now, "Registrar lastExpiringCertNotificationSentDate cannot be null");
|
||||
getInstance().lastExpiringCertNotificationSentDate = now;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setLastExpiringFailoverCertNotificationSentDate(DateTime now) {
|
||||
checkArgumentNotNull(
|
||||
now, "Registrar lastExpiringFailoverCertNotificationSentDate cannot be null");
|
||||
getInstance().lastExpiringFailoverCertNotificationSentDate = now;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setFailoverClientCertificate(String clientCertificate, DateTime now) {
|
||||
clientCertificate = emptyToNull(clientCertificate);
|
||||
String clientCertificateHash = calculateHash(clientCertificate);
|
||||
|
||||
@@ -47,7 +47,6 @@ import com.googlecode.objectify.annotation.Embed;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Mapify;
|
||||
import com.googlecode.objectify.annotation.OnLoad;
|
||||
import com.googlecode.objectify.annotation.OnSave;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.Buildable;
|
||||
@@ -112,26 +111,6 @@ public class Registry extends ImmutableObject implements Buildable, DatastoreAnd
|
||||
@PostLoad
|
||||
void postLoad() {
|
||||
tldStr = tldStrId;
|
||||
// TODO(sarahbot@): Remove the rest of this method after this data migration is complete
|
||||
if (premiumListName != null) {
|
||||
premiumList = Key.create(getCrossTldKey(), PremiumList.class, premiumListName);
|
||||
}
|
||||
if (reservedListNames != null) {
|
||||
reservedLists =
|
||||
reservedListNames.stream()
|
||||
.map(name -> Key.create(getCrossTldKey(), ReservedList.class, name))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(sarahbot@): Remove this method after this data migration is complete
|
||||
@OnLoad
|
||||
void onLoad() {
|
||||
if (reservedLists != null) {
|
||||
reservedListNames =
|
||||
reservedLists.stream().map(key -> key.getName()).collect(toImmutableSet());
|
||||
}
|
||||
premiumListName = premiumList == null ? null : premiumList.getName();
|
||||
}
|
||||
|
||||
/** The suffix that identifies roids as belonging to this specific tld, e.g. -HOW for .how. */
|
||||
@@ -408,9 +387,6 @@ public class Registry extends ImmutableObject implements Buildable, DatastoreAnd
|
||||
@Column(nullable = false)
|
||||
CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
|
||||
|
||||
/** The set of reserved lists that are applicable to this registry. */
|
||||
@Transient Set<Key<ReservedList>> reservedLists;
|
||||
|
||||
/** The set of reserved list names that are applicable to this registry. */
|
||||
@Column(name = "reserved_list_names")
|
||||
Set<String> reservedListNames;
|
||||
@@ -423,13 +399,10 @@ public class Registry extends ImmutableObject implements Buildable, DatastoreAnd
|
||||
* for a registry, the database should be queried for the entity with this name that has the
|
||||
* largest revision ID.
|
||||
*/
|
||||
public ImmutableSet<Key<ReservedList>> getReservedLists() {
|
||||
return nullToEmptyImmutableCopy(reservedLists);
|
||||
public ImmutableSet<String> getReservedListNames() {
|
||||
return nullToEmptyImmutableCopy(reservedListNames);
|
||||
}
|
||||
|
||||
/** The static {@link PremiumList} for this TLD, if there is one. */
|
||||
@Transient Key<PremiumList> premiumList;
|
||||
|
||||
/**
|
||||
* The name of the {@link PremiumList} for this TLD, if there is one.
|
||||
*
|
||||
@@ -647,8 +620,8 @@ public class Registry extends ImmutableObject implements Buildable, DatastoreAnd
|
||||
return anchorTenantAddGracePeriodLength;
|
||||
}
|
||||
|
||||
public Optional<Key<PremiumList>> getPremiumList() {
|
||||
return Optional.ofNullable(premiumList);
|
||||
public Optional<String> getPremiumListName() {
|
||||
return Optional.ofNullable(premiumListName);
|
||||
}
|
||||
|
||||
public CurrencyUnit getCurrency() {
|
||||
@@ -919,26 +892,15 @@ public class Registry extends ImmutableObject implements Buildable, DatastoreAnd
|
||||
|
||||
public Builder setReservedLists(Set<ReservedList> reservedLists) {
|
||||
checkArgumentNotNull(reservedLists, "reservedLists must not be null");
|
||||
ImmutableSet.Builder<Key<ReservedList>> builder = new ImmutableSet.Builder<>();
|
||||
ImmutableSet.Builder<String> nameBuilder = new ImmutableSet.Builder<>();
|
||||
for (ReservedList reservedList : reservedLists) {
|
||||
builder.add(Key.create(reservedList));
|
||||
nameBuilder.add(reservedList.getName());
|
||||
}
|
||||
getInstance().reservedLists = builder.build();
|
||||
getInstance().reservedListNames = nameBuilder.build();
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setPremiumList(@Nullable PremiumList premiumList) {
|
||||
getInstance().premiumList = (premiumList == null) ? null : Key.create(premiumList);
|
||||
getInstance().premiumListName = (premiumList == null) ? null : premiumList.getName();
|
||||
return this;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public Builder setPremiumListKey(@Nullable Key<PremiumList> premiumList) {
|
||||
getInstance().premiumList = premiumList;
|
||||
getInstance().premiumListName = (premiumList == null) ? null : premiumList.getName();
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -164,15 +164,12 @@ public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends Dom
|
||||
|
||||
/** Gets the names of the tlds that reference this list. */
|
||||
public final ImmutableSet<String> getReferencingTlds() {
|
||||
Key<? extends BaseDomainLabelList<?, ?>> key = Key.create(this);
|
||||
return getTlds()
|
||||
.stream()
|
||||
.filter((tld) -> refersToKey(Registry.get(tld), key))
|
||||
return getTlds().stream()
|
||||
.filter((tld) -> refersToList(Registry.get(tld), name))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
|
||||
protected abstract boolean refersToKey(
|
||||
Registry registry, Key<? extends BaseDomainLabelList<?, ?>> key);
|
||||
protected abstract boolean refersToList(Registry registry, String name);
|
||||
|
||||
protected static <R> Optional<R> getFromCache(String listName, LoadingCache<String, R> cache) {
|
||||
try {
|
||||
|
||||
@@ -282,8 +282,8 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean refersToKey(Registry registry, Key<? extends BaseDomainLabelList<?, ?>> key) {
|
||||
return Objects.equals(registry.getPremiumList().orElse(null), key);
|
||||
public boolean refersToList(Registry registry, String name) {
|
||||
return Objects.equals(registry.getPremiumListName().orElse(null), name);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -20,7 +20,6 @@ import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static com.google.common.collect.ImmutableMap.toImmutableMap;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
|
||||
import static google.registry.model.ImmutableObject.Insignificant;
|
||||
import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
|
||||
import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
@@ -34,7 +33,6 @@ import com.google.common.cache.LoadingCache;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Embed;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Mapify;
|
||||
@@ -186,8 +184,8 @@ public final class ReservedList
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean refersToKey(Registry registry, Key<? extends BaseDomainLabelList<?, ?>> key) {
|
||||
return registry.getReservedLists().contains(key);
|
||||
protected boolean refersToList(Registry registry, String name) {
|
||||
return registry.getReservedListNames().contains(name);
|
||||
}
|
||||
|
||||
/** Determines whether the ReservedList is in use on any Registry */
|
||||
@@ -236,11 +234,6 @@ public final class ReservedList
|
||||
return getFromCache(listName, cache);
|
||||
}
|
||||
|
||||
/** Loads a ReservedList from its Objectify key. */
|
||||
public static Optional<ReservedList> load(Key<ReservedList> key) {
|
||||
return get(key.getName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Queries the set of all reserved lists associated with the specified TLD and returns the
|
||||
* reservation types of the label.
|
||||
@@ -270,7 +263,7 @@ public final class ReservedList
|
||||
new ImmutableSet.Builder<>();
|
||||
|
||||
// Loop through all reservation lists and add each of them.
|
||||
for (ReservedList rl : loadReservedLists(registry.getReservedLists())) {
|
||||
for (ReservedList rl : loadReservedLists(registry.getReservedListNames())) {
|
||||
if (rl.getReservedListEntries().containsKey(label)) {
|
||||
ReservedListEntry entry = rl.getReservedListEntries().get(label);
|
||||
entriesBuilder.add(entry);
|
||||
@@ -285,17 +278,15 @@ public final class ReservedList
|
||||
}
|
||||
|
||||
private static ImmutableSet<ReservedList> loadReservedLists(
|
||||
ImmutableSet<Key<ReservedList>> reservedListKeys) {
|
||||
return reservedListKeys
|
||||
.stream()
|
||||
ImmutableSet<String> reservedListNames) {
|
||||
return reservedListNames.stream()
|
||||
.map(
|
||||
(listKey) -> {
|
||||
(listName) -> {
|
||||
try {
|
||||
return cache.get(listKey.getName());
|
||||
return cache.get(listName);
|
||||
} catch (ExecutionException e) {
|
||||
throw new UncheckedExecutionException(
|
||||
String.format(
|
||||
"Could not load the reserved list '%s' from the cache", listKey.getName()),
|
||||
String.format("Could not load the reserved list '%s' from the cache", listName),
|
||||
e);
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.InCrossTld;
|
||||
import google.registry.model.annotations.ReportedOn;
|
||||
import google.registry.model.common.EntityGroupRoot;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
|
||||
/** Pointer to the latest {@link KmsSecretRevision}. */
|
||||
@Entity
|
||||
@ReportedOn
|
||||
@InCrossTld
|
||||
public class KmsSecret extends ImmutableObject implements DatastoreOnlyEntity {
|
||||
|
||||
/** The unique name of this {@link KmsSecret}. */
|
||||
@Id String name;
|
||||
|
||||
@Parent Key<EntityGroupRoot> parent = getCrossTldKey();
|
||||
|
||||
/** The pointer to the latest {@link KmsSecretRevision}. */
|
||||
Key<KmsSecretRevision> latestRevision;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public Key<KmsSecretRevision> getLatestRevision() {
|
||||
return latestRevision;
|
||||
}
|
||||
|
||||
public static KmsSecret create(String name, KmsSecretRevision latestRevision) {
|
||||
KmsSecret instance = new KmsSecret();
|
||||
instance.name = name;
|
||||
instance.latestRevision = Key.create(latestRevision);
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Ignore;
|
||||
import com.googlecode.objectify.annotation.OnLoad;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.Buildable;
|
||||
import google.registry.model.CreateAutoTimestamp;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.InCrossTld;
|
||||
import google.registry.model.annotations.ReportedOn;
|
||||
import google.registry.schema.replay.NonReplicatedEntity;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Index;
|
||||
import javax.persistence.PostLoad;
|
||||
import javax.persistence.Table;
|
||||
import javax.persistence.Transient;
|
||||
|
||||
/**
|
||||
* An encrypted value.
|
||||
*
|
||||
* <p>Used to store passwords and other sensitive information in Datastore. Multiple versions of a
|
||||
* {@link KmsSecretRevision} may be persisted but only the latest version is primary. A key to the
|
||||
* primary version is stored by {@link KmsSecret#latestRevision}.
|
||||
*
|
||||
* <p>The value can be encrypted and decrypted using Cloud KMS.
|
||||
*
|
||||
* <p>Note that the primary key of this entity is {@link #revisionKey}, which is auto-generated by
|
||||
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly
|
||||
* succeeds, we will end up with having two exact same revisions that differ only by revisionKey.
|
||||
* This is fine though, because we only use the revision with the highest revisionKey.
|
||||
*
|
||||
* <p>TODO(b/177567432): remove Datastore-specific fields post-Registry-3.0-migration and rename to
|
||||
* KmsSecret.
|
||||
*
|
||||
* @see <a href="https://cloud.google.com/kms/docs/">Google Cloud Key Management Service
|
||||
* Documentation</a>
|
||||
* @see google.registry.keyring.kms.KmsKeyring
|
||||
*/
|
||||
@Entity
|
||||
@ReportedOn
|
||||
@javax.persistence.Entity(name = "KmsSecret")
|
||||
@Table(indexes = {@Index(columnList = "secretName")})
|
||||
@InCrossTld
|
||||
public class KmsSecretRevision extends ImmutableObject implements NonReplicatedEntity {
|
||||
|
||||
/**
|
||||
* The maximum allowable secret size. Although Datastore allows entities up to 1 MB in size,
|
||||
* BigQuery imports of Datastore backups limit individual columns (entity attributes) to 64 KB.
|
||||
*/
|
||||
private static final int MAX_SECRET_SIZE_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* The revision of this secret.
|
||||
*
|
||||
* <p>TODO(b/177567432): change name of the variable to revisionId once we're off Datastore
|
||||
*/
|
||||
@Id
|
||||
@javax.persistence.Id
|
||||
@Column(name = "revisionId")
|
||||
long revisionKey;
|
||||
|
||||
/** The parent {@link KmsSecret} which contains metadata about this {@link KmsSecretRevision}. */
|
||||
@Parent @Transient Key<KmsSecret> parent;
|
||||
@Column(nullable = false)
|
||||
@Ignore
|
||||
String secretName;
|
||||
|
||||
/**
|
||||
* The name of the {@code cryptoKeyVersion} associated with this {@link KmsSecretRevision}.
|
||||
*
|
||||
* <p>TODO: change name of the variable to cryptoKeyVersionName once we're off Datastore
|
||||
*
|
||||
* @see <a
|
||||
* href="https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys.cryptoKeyVersions">projects.locations.keyRings.cryptoKeys.cryptoKeyVersions</a>
|
||||
*/
|
||||
@Column(nullable = false, name = "cryptoKeyVersionName")
|
||||
String kmsCryptoKeyVersionName;
|
||||
|
||||
/**
|
||||
* The base64-encoded encrypted value of this {@link KmsSecretRevision} as returned by the Cloud
|
||||
* KMS API.
|
||||
*
|
||||
* @see <a
|
||||
* href="https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys/encrypt">projects.locations.keyRings.cryptoKeys.encrypt</a>
|
||||
*/
|
||||
@Column(nullable = false)
|
||||
String encryptedValue;
|
||||
|
||||
/** An automatically managed creation timestamp. */
|
||||
@Column(nullable = false)
|
||||
CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
|
||||
|
||||
public String getKmsCryptoKeyVersionName() {
|
||||
return kmsCryptoKeyVersionName;
|
||||
}
|
||||
|
||||
public String getEncryptedValue() {
|
||||
return encryptedValue;
|
||||
}
|
||||
|
||||
// When loading from SQL, fill out the Datastore-specific field
|
||||
@PostLoad
|
||||
void postLoad() {
|
||||
parent = Key.create(getCrossTldKey(), KmsSecret.class, secretName);
|
||||
}
|
||||
|
||||
// When loading from Datastore, fill out the SQL-specific field
|
||||
@OnLoad
|
||||
void onLoad() {
|
||||
secretName = parent.getName();
|
||||
}
|
||||
|
||||
/** A builder for constructing {@link KmsSecretRevision} entities, since they are immutable. */
|
||||
public static class Builder extends Buildable.Builder<KmsSecretRevision> {
|
||||
|
||||
public Builder setKmsCryptoKeyVersionName(String kmsCryptoKeyVersionName) {
|
||||
getInstance().kmsCryptoKeyVersionName = kmsCryptoKeyVersionName;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEncryptedValue(String encryptedValue) {
|
||||
checkArgument(
|
||||
encryptedValue.length() <= MAX_SECRET_SIZE_BYTES,
|
||||
"Secret is greater than %s bytes",
|
||||
MAX_SECRET_SIZE_BYTES);
|
||||
|
||||
getInstance().encryptedValue = encryptedValue;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the parent {@link KmsSecret}.
|
||||
*
|
||||
* <p>The secret may not exist yet, so it is referred to by name rather than by reference.
|
||||
*/
|
||||
public Builder setParent(String secretName) {
|
||||
getInstance().parent = Key.create(getCrossTldKey(), KmsSecret.class, secretName);
|
||||
getInstance().secretName = secretName;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* A {@link KmsSecretRevision} DAO for Cloud SQL.
|
||||
*
|
||||
* <p>TODO(b/177567432): Rename this class to KmsSecretDao after migrating to Cloud SQL.
|
||||
*/
|
||||
public class KmsSecretRevisionSqlDao {
|
||||
|
||||
private KmsSecretRevisionSqlDao() {}
|
||||
|
||||
/** Saves the given KMS secret revision. */
|
||||
public static void save(KmsSecretRevision kmsSecretRevision) {
|
||||
checkArgumentNotNull(kmsSecretRevision, "kmsSecretRevision cannot be null");
|
||||
jpaTm().assertInTransaction();
|
||||
jpaTm().put(kmsSecretRevision);
|
||||
}
|
||||
|
||||
/** Returns the latest revision for the secret name given, or absent if nonexistent. */
|
||||
public static Optional<KmsSecretRevision> getLatestRevision(String secretName) {
|
||||
checkArgument(!isNullOrEmpty(secretName), "secretName cannot be null or empty");
|
||||
jpaTm().assertInTransaction();
|
||||
return jpaTm()
|
||||
.query(
|
||||
"FROM KmsSecret ks WHERE ks.revisionKey IN (SELECT MAX(revisionKey) FROM "
|
||||
+ "KmsSecret subKs WHERE subKs.secretName = :secretName)",
|
||||
KmsSecretRevision.class)
|
||||
.setParameter("secretName", secretName)
|
||||
.getResultStream()
|
||||
.findFirst();
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,6 @@ import google.registry.export.datastore.DatastoreAdminModule;
|
||||
import google.registry.export.sheet.SheetsServiceModule;
|
||||
import google.registry.flows.ServerTridProviderModule;
|
||||
import google.registry.flows.custom.CustomLogicFactoryModule;
|
||||
import google.registry.gcs.GcsServiceModule;
|
||||
import google.registry.groups.DirectoryModule;
|
||||
import google.registry.groups.GroupsModule;
|
||||
import google.registry.groups.GroupssettingsModule;
|
||||
@@ -64,7 +63,6 @@ import javax.inject.Singleton;
|
||||
DirectoryModule.class,
|
||||
DummyKeyringModule.class,
|
||||
DriveModule.class,
|
||||
GcsServiceModule.class,
|
||||
GroupsModule.class,
|
||||
GroupssettingsModule.class,
|
||||
JSchModule.class,
|
||||
|
||||
@@ -22,7 +22,6 @@ import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.export.DriveModule;
|
||||
import google.registry.flows.ServerTridProviderModule;
|
||||
import google.registry.flows.custom.CustomLogicFactoryModule;
|
||||
import google.registry.gcs.GcsServiceModule;
|
||||
import google.registry.groups.DirectoryModule;
|
||||
import google.registry.groups.GroupsModule;
|
||||
import google.registry.groups.GroupssettingsModule;
|
||||
@@ -54,7 +53,6 @@ import javax.inject.Singleton;
|
||||
DirectoryModule.class,
|
||||
DummyKeyringModule.class,
|
||||
DriveModule.class,
|
||||
GcsServiceModule.class,
|
||||
GroupsModule.class,
|
||||
GroupssettingsModule.class,
|
||||
Jackson2Module.class,
|
||||
|
||||
@@ -21,6 +21,7 @@ import google.registry.model.billing.BillingEvent.OneTime;
|
||||
import google.registry.model.billing.BillingEvent.Recurring;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import java.io.Serializable;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.persistence.AttributeOverride;
|
||||
import javax.persistence.AttributeOverrides;
|
||||
@@ -46,7 +47,7 @@ public abstract class BillingVKey<K> extends EppHistoryVKey<K, DomainBase> {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object createSqlKey() {
|
||||
public Serializable createSqlKey() {
|
||||
return billingId;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import java.io.Serializable;
|
||||
import javax.persistence.Embeddable;
|
||||
|
||||
/** {@link VKey} for {@link HistoryEntry} which parent is {@link DomainBase}. */
|
||||
@@ -35,7 +36,7 @@ public class DomainHistoryVKey extends EppHistoryVKey<HistoryEntry, DomainBase>
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object createSqlKey() {
|
||||
public Serializable createSqlKey() {
|
||||
return new DomainHistoryId(repoId, historyRevisionId);
|
||||
}
|
||||
|
||||
|
||||
@@ -101,7 +101,7 @@ public abstract class EppHistoryVKey<K, E extends EppResource> extends Immutable
|
||||
return VKey.create(vKeyType, createSqlKey(), createOfyKey());
|
||||
}
|
||||
|
||||
public abstract Object createSqlKey();
|
||||
public abstract Serializable createSqlKey();
|
||||
|
||||
public abstract Key<K> createOfyKey();
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
private static final long serialVersionUID = -5291472863840231240L;
|
||||
|
||||
// The SQL key for the referenced entity.
|
||||
Object sqlKey;
|
||||
Serializable sqlKey;
|
||||
|
||||
// The objectify key for the referenced entity.
|
||||
Key<T> ofyKey;
|
||||
@@ -46,7 +46,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
|
||||
VKey() {}
|
||||
|
||||
VKey(Class<? extends T> kind, Key<T> ofyKey, Object sqlKey) {
|
||||
VKey(Class<? extends T> kind, Key<T> ofyKey, Serializable sqlKey) {
|
||||
this.kind = kind;
|
||||
this.ofyKey = ofyKey;
|
||||
this.sqlKey = sqlKey;
|
||||
@@ -57,7 +57,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
*
|
||||
* <p>Deprecated. Create symmetric keys with create() instead.
|
||||
*/
|
||||
public static <T> VKey<T> createSql(Class<T> kind, Object sqlKey) {
|
||||
public static <T> VKey<T> createSql(Class<T> kind, Serializable sqlKey) {
|
||||
checkArgumentNotNull(kind, "kind must not be null");
|
||||
checkArgumentNotNull(sqlKey, "sqlKey must not be null");
|
||||
return new VKey<T>(kind, null, sqlKey);
|
||||
@@ -71,7 +71,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
}
|
||||
|
||||
/** Creates a {@link VKey} which only contains both sql and ofy primary key. */
|
||||
public static <T> VKey<T> create(Class<T> kind, Object sqlKey, Key<T> ofyKey) {
|
||||
public static <T> VKey<T> create(Class<T> kind, Serializable sqlKey, Key<T> ofyKey) {
|
||||
checkArgumentNotNull(kind, "kind must not be null");
|
||||
checkArgumentNotNull(sqlKey, "sqlKey must not be null");
|
||||
checkArgumentNotNull(ofyKey, "ofyKey must not be null");
|
||||
@@ -84,8 +84,8 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
* <p>IMPORTANT USAGE NOTE: Datastore entities that are not roots of entity groups (i.e. those
|
||||
* that do not have a null parent in their Objectify keys) require the full entity group
|
||||
* inheritance chain to be specified and thus cannot use this create method. You need to use
|
||||
* {@link #create(Class, Object, Key)} instead and pass in the full, valid parent field in the
|
||||
* Datastore key.
|
||||
* {@link #create(Class, Serializable, Key)} instead and pass in the full, valid parent field in
|
||||
* the Datastore key.
|
||||
*/
|
||||
public static <T> VKey<T> create(Class<T> kind, long id) {
|
||||
checkArgument(
|
||||
@@ -102,8 +102,8 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
* <p>IMPORTANT USAGE NOTE: Datastore entities that are not roots of entity groups (i.e. those
|
||||
* that do not have a null parent in their Objectify keys) require the full entity group
|
||||
* inheritance chain to be specified and thus cannot use this create method. You need to use
|
||||
* {@link #create(Class, Object, Key)} instead and pass in the full, valid parent field in the
|
||||
* Datastore key.
|
||||
* {@link #create(Class, Serializable, Key)} instead and pass in the full, valid parent field in
|
||||
* the Datastore key.
|
||||
*/
|
||||
public static <T> VKey<T> create(Class<T> kind, String name) {
|
||||
checkArgument(
|
||||
@@ -172,7 +172,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
throw new IllegalArgumentException("Missing value for last key of type " + lastClass);
|
||||
}
|
||||
|
||||
Object sqlKey = getSqlKey();
|
||||
Serializable sqlKey = getSqlKey();
|
||||
Key<T> ofyKey =
|
||||
sqlKey instanceof Long
|
||||
? Key.create(lastKey, getKind(), (Long) sqlKey)
|
||||
@@ -197,7 +197,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
|
||||
}
|
||||
|
||||
/** Returns the SQL primary key. */
|
||||
public Object getSqlKey() {
|
||||
public Serializable getSqlKey() {
|
||||
checkState(sqlKey != null, "Attempting obtain a null SQL key.");
|
||||
return this.sqlKey;
|
||||
}
|
||||
|
||||
@@ -16,11 +16,13 @@ package google.registry.persistence.converter;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.persistence.VKey;
|
||||
import java.io.Serializable;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.persistence.AttributeConverter;
|
||||
|
||||
/** Converts VKey to a string column. */
|
||||
public abstract class VKeyConverter<T, C> implements AttributeConverter<VKey<? extends T>, C> {
|
||||
public abstract class VKeyConverter<T, C extends Serializable>
|
||||
implements AttributeConverter<VKey<? extends T>, C> {
|
||||
@Override
|
||||
@Nullable
|
||||
public C convertToDatabaseColumn(@Nullable VKey<? extends T> attribute) {
|
||||
|
||||
@@ -62,6 +62,13 @@ public class CriteriaQueryBuilder<T> {
|
||||
return where(root.get(fieldName).in(values));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a WHERE clause to the query specifying that a value must not be in the given collection.
|
||||
*/
|
||||
public CriteriaQueryBuilder<T> whereFieldIsNotIn(String fieldName, Collection<?> values) {
|
||||
return where(root.get(fieldName).in(values).not());
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a WHERE clause to the query specifying that a collection field must contain a particular
|
||||
* value.
|
||||
|
||||
+28
-6
@@ -30,14 +30,14 @@ import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
|
||||
import google.registry.model.ofy.DatastoreTransactionManager;
|
||||
import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
|
||||
import google.registry.persistence.JpaRetries;
|
||||
import google.registry.persistence.VKey;
|
||||
@@ -46,6 +46,7 @@ import google.registry.schema.replay.SqlOnlyEntity;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.util.SystemSleeper;
|
||||
import java.io.Serializable;
|
||||
import java.lang.reflect.Array;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.Calendar;
|
||||
@@ -93,8 +94,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
EppResourceIndex.class,
|
||||
ForeignKeyContactIndex.class,
|
||||
ForeignKeyDomainIndex.class,
|
||||
ForeignKeyHostIndex.class,
|
||||
KmsSecret.class);
|
||||
ForeignKeyHostIndex.class);
|
||||
|
||||
// EntityManagerFactory is thread safe.
|
||||
private final EntityManagerFactory emf;
|
||||
@@ -104,6 +104,10 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
// synchronously.
|
||||
private final ThreadLocal<TransactionInfo> transactionInfo =
|
||||
ThreadLocal.withInitial(TransactionInfo::new);
|
||||
// If this value is present, use it to determine whether or not to replay SQL transactions to
|
||||
// Datastore, rather than using the schedule stored in Datastore.
|
||||
private static ThreadLocal<Optional<Boolean>> replaySqlToDatastoreOverrideForTest =
|
||||
ThreadLocal.withInitial(Optional::empty);
|
||||
|
||||
public JpaTransactionManagerImpl(EntityManagerFactory emf, Clock clock) {
|
||||
this.emf = emf;
|
||||
@@ -489,7 +493,8 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
loadByKey(
|
||||
VKey.createSql(
|
||||
possibleChild.getClass(),
|
||||
emf.getPersistenceUnitUtil().getIdentifier(possibleChild)));
|
||||
// Casting to Serializable is safe according to JPA (JSR 338 sec. 2.4).
|
||||
(Serializable) emf.getPersistenceUnitUtil().getIdentifier(possibleChild)));
|
||||
return returnValue;
|
||||
}
|
||||
|
||||
@@ -736,6 +741,16 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
return entity;
|
||||
}
|
||||
|
||||
/** Sets the override to always/never replay SQL transactions to Datastore. */
|
||||
public static void setReplaySqlToDatastoreOverrideForTest(boolean replaySqlToDs) {
|
||||
replaySqlToDatastoreOverrideForTest.set(Optional.of(replaySqlToDs));
|
||||
}
|
||||
|
||||
/** Removes the replay-SQL-to-Datastore override; the migration schedule will then be used. */
|
||||
public static void removeReplaySqlToDsOverrideForTest() {
|
||||
replaySqlToDatastoreOverrideForTest.set(Optional.empty());
|
||||
}
|
||||
|
||||
private static class TransactionInfo {
|
||||
EntityManager entityManager;
|
||||
boolean inTransaction = false;
|
||||
@@ -755,7 +770,14 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
checkArgumentNotNull(clock);
|
||||
inTransaction = true;
|
||||
transactionTime = clock.nowUtc();
|
||||
if (withBackup && RegistryConfig.getCloudSqlReplicateTransactions()) {
|
||||
if (withBackup
|
||||
&& replaySqlToDatastoreOverrideForTest
|
||||
.get()
|
||||
.orElseGet(
|
||||
() ->
|
||||
DatabaseMigrationStateSchedule.getValueAtTime(transactionTime)
|
||||
.getReplayDirection()
|
||||
.equals(ReplayDirection.SQL_TO_DATASTORE))) {
|
||||
contentsBuilder = new Transaction.Builder();
|
||||
}
|
||||
}
|
||||
|
||||
+31
-6
@@ -16,17 +16,22 @@ package google.registry.persistence.transaction;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.api.utils.SystemProperty;
|
||||
import com.google.appengine.api.utils.SystemProperty.Environment.Value;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Suppliers;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
|
||||
import google.registry.model.ofy.DatastoreTransactionManager;
|
||||
import google.registry.persistence.DaggerPersistenceComponent;
|
||||
import google.registry.tools.RegistryToolEnvironment;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Factory class to create {@link TransactionManager} instance. */
|
||||
// TODO: Rename this to PersistenceFactory and move to persistence package.
|
||||
@@ -34,7 +39,8 @@ public class TransactionManagerFactory {
|
||||
|
||||
private static final DatastoreTransactionManager ofyTm = createTransactionManager();
|
||||
|
||||
@NonFinalForTesting private static TransactionManager tm = ofyTm;
|
||||
/** Optional override to manually set the transaction manager per-test. */
|
||||
private static Optional<TransactionManager> tmForTest = Optional.empty();
|
||||
|
||||
/** Supplier for jpaTm so that it is initialized only once, upon first usage. */
|
||||
@NonFinalForTesting
|
||||
@@ -70,9 +76,19 @@ public class TransactionManagerFactory {
|
||||
return SystemProperty.environment.value() == Value.Production;
|
||||
}
|
||||
|
||||
/** Returns {@link TransactionManager} instance. */
|
||||
/**
|
||||
* Returns the {@link TransactionManager} instance.
|
||||
*
|
||||
* <p>Returns the {@link JpaTransactionManager} or {@link DatastoreTransactionManager} based on
|
||||
* the migration schedule or the manually specified per-test transaction manager.
|
||||
*/
|
||||
public static TransactionManager tm() {
|
||||
return tm;
|
||||
if (tmForTest.isPresent()) {
|
||||
return tmForTest.get();
|
||||
}
|
||||
PrimaryDatabase primaryDatabase =
|
||||
DatabaseMigrationStateSchedule.getValueAtTime(DateTime.now(UTC)).getPrimaryDatabase();
|
||||
return primaryDatabase.equals(PrimaryDatabase.DATASTORE) ? ofyTm() : jpaTm();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -111,9 +127,18 @@ public class TransactionManagerFactory {
|
||||
jpaTm = Suppliers.memoize(jpaTmSupplier::get);
|
||||
}
|
||||
|
||||
/** Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}. */
|
||||
/**
|
||||
* Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}.
|
||||
*
|
||||
* <p>Used when overriding the per-test transaction manager for dual-database tests.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
public static void setTm(TransactionManager newTm) {
|
||||
tm = newTm;
|
||||
public static void setTmForTest(TransactionManager newTm) {
|
||||
tmForTest = Optional.of(newTm);
|
||||
}
|
||||
|
||||
/** Resets the overridden transaction manager post-test. */
|
||||
public static void removeTmOverrideForTest() {
|
||||
tmForTest = Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ package google.registry.rde;
|
||||
import static google.registry.model.rde.RdeMode.THIN;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
@@ -85,10 +85,10 @@ public final class BrdaCopyAction implements Runnable {
|
||||
|
||||
private void copyAsRyde() throws IOException {
|
||||
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
|
||||
GcsFilename xmlFilename = new GcsFilename(stagingBucket, prefix + ".xml.ghostryde");
|
||||
GcsFilename xmlLengthFilename = new GcsFilename(stagingBucket, prefix + ".xml.length");
|
||||
GcsFilename rydeFile = new GcsFilename(brdaBucket, prefix + ".ryde");
|
||||
GcsFilename sigFile = new GcsFilename(brdaBucket, prefix + ".sig");
|
||||
BlobId xmlFilename = BlobId.of(stagingBucket, prefix + ".xml.ghostryde");
|
||||
BlobId xmlLengthFilename = BlobId.of(stagingBucket, prefix + ".xml.length");
|
||||
BlobId rydeFile = BlobId.of(brdaBucket, prefix + ".ryde");
|
||||
BlobId sigFile = BlobId.of(brdaBucket, prefix + ".sig");
|
||||
|
||||
long xmlLength = readXmlLength(xmlLengthFilename);
|
||||
|
||||
@@ -107,7 +107,7 @@ public final class BrdaCopyAction implements Runnable {
|
||||
}
|
||||
|
||||
/** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */
|
||||
private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException {
|
||||
private long readXmlLength(BlobId xmlLengthFilename) throws IOException {
|
||||
try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) {
|
||||
return Ghostryde.readLength(input);
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ public abstract class PendingDeposit implements Serializable {
|
||||
return new AutoValue_PendingDeposit(false, tld, watermark, mode, cursor, interval, null, null);
|
||||
}
|
||||
|
||||
static PendingDeposit createInManualOperation(
|
||||
public static PendingDeposit createInManualOperation(
|
||||
String tld,
|
||||
DateTime watermark,
|
||||
RdeMode mode,
|
||||
|
||||
@@ -23,7 +23,7 @@ import static google.registry.persistence.transaction.TransactionManagerUtil.tra
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
@@ -97,7 +97,7 @@ public final class RdeReportAction implements Runnable, EscrowTask {
|
||||
.orElseThrow(
|
||||
() -> new IllegalStateException("RdeRevision was not set on generated deposit"));
|
||||
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
|
||||
GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
|
||||
BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde");
|
||||
verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
|
||||
reporter.send(readReportFromGcs(reportFilename));
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
@@ -106,7 +106,7 @@ public final class RdeReportAction implements Runnable, EscrowTask {
|
||||
}
|
||||
|
||||
/** Reads and decrypts the XML file from cloud storage. */
|
||||
private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException {
|
||||
private byte[] readReportFromGcs(BlobId reportFilename) throws IOException {
|
||||
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename);
|
||||
InputStream ghostrydeDecoder = Ghostryde.decoder(gcsInput, stagingDecryptionKey)) {
|
||||
return ByteStreams.toByteArray(ghostrydeDecoder);
|
||||
|
||||
@@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.Multimaps;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.mapreduce.inputs.NullInput;
|
||||
@@ -206,6 +207,7 @@ public final class RdeStagingAction implements Runnable {
|
||||
@Inject PendingDepositChecker pendingDepositChecker;
|
||||
@Inject RdeStagingReducer.Factory reducerFactory;
|
||||
@Inject Response response;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject @Config("transactionCooldown") Duration transactionCooldown;
|
||||
@Inject @Parameter(RdeModule.PARAM_MANUAL) boolean manual;
|
||||
@@ -234,7 +236,7 @@ public final class RdeStagingAction implements Runnable {
|
||||
}
|
||||
ValidationMode validationMode = lenient ? LENIENT : STRICT;
|
||||
RdeStagingMapper mapper = new RdeStagingMapper(validationMode, pendings);
|
||||
RdeStagingReducer reducer = reducerFactory.create(validationMode);
|
||||
RdeStagingReducer reducer = reducerFactory.create(validationMode, gcsUtils);
|
||||
|
||||
mrRunner
|
||||
.setJobName("Stage escrow deposits for all TLDs")
|
||||
|
||||
@@ -16,7 +16,6 @@ package google.registry.rde;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
|
||||
@@ -24,10 +23,9 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
@@ -69,27 +67,27 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
|
||||
private final TaskQueueUtils taskQueueUtils;
|
||||
private final LockHandler lockHandler;
|
||||
private final int gcsBufferSize;
|
||||
private final String bucket;
|
||||
private final Duration lockTimeout;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final RdeMarshaller marshaller;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
RdeStagingReducer(
|
||||
TaskQueueUtils taskQueueUtils,
|
||||
LockHandler lockHandler,
|
||||
int gcsBufferSize,
|
||||
String bucket,
|
||||
Duration lockTimeout,
|
||||
byte[] stagingKeyBytes,
|
||||
ValidationMode validationMode) {
|
||||
ValidationMode validationMode,
|
||||
GcsUtils gcsUtils) {
|
||||
this.taskQueueUtils = taskQueueUtils;
|
||||
this.lockHandler = lockHandler;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.bucket = bucket;
|
||||
this.lockTimeout = lockTimeout;
|
||||
this.stagingKeyBytes = stagingKeyBytes;
|
||||
this.marshaller = new RdeMarshaller(validationMode);
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -113,8 +111,6 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
|
||||
// Construct things that Dagger would inject if this wasn't serialized.
|
||||
PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
RdeCounter counter = new RdeCounter();
|
||||
|
||||
// Determine some basic things about the deposit.
|
||||
@@ -130,14 +126,14 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
|
||||
prefix = "manual/" + key.directoryWithTrailingSlash() + prefix;
|
||||
}
|
||||
GcsFilename xmlFilename = new GcsFilename(bucket, prefix + ".xml.ghostryde");
|
||||
BlobId xmlFilename = BlobId.of(bucket, prefix + ".xml.ghostryde");
|
||||
// This file will contain the byte length (ASCII) of the raw unencrypted XML.
|
||||
//
|
||||
// This is necessary because RdeUploadAction creates a tar file which requires that the length
|
||||
// be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
|
||||
// length, so we just save it separately.
|
||||
GcsFilename xmlLengthFilename = new GcsFilename(bucket, prefix + ".xml.length");
|
||||
GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
|
||||
BlobId xmlLengthFilename = BlobId.of(bucket, prefix + ".xml.length");
|
||||
BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde");
|
||||
|
||||
// These variables will be populated as we write the deposit XML and used for other files.
|
||||
boolean failed = false;
|
||||
@@ -146,8 +142,8 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
// Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
|
||||
|
||||
logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = cloudStorage.openOutputStream(xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
|
||||
Writer output = new OutputStreamWriter(ghostrydeEncoder, UTF_8)) {
|
||||
|
||||
@@ -194,7 +190,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
// This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
|
||||
if (mode == RdeMode.FULL) {
|
||||
logger.atInfo().log("Writing %s", reportFilename);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(reportFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
|
||||
counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
|
||||
} catch (IOException | XmlException e) {
|
||||
@@ -249,22 +245,21 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
||||
static class Factory {
|
||||
@Inject TaskQueueUtils taskQueueUtils;
|
||||
@Inject LockHandler lockHandler;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject @Config("rdeBucket") String bucket;
|
||||
@Inject @Config("rdeStagingLockTimeout") Duration lockTimeout;
|
||||
@Inject @KeyModule.Key("rdeStagingEncryptionKey") byte[] stagingKeyBytes;
|
||||
|
||||
@Inject Factory() {}
|
||||
|
||||
RdeStagingReducer create(ValidationMode validationMode) {
|
||||
RdeStagingReducer create(ValidationMode validationMode, GcsUtils gcsUtils) {
|
||||
return new RdeStagingReducer(
|
||||
taskQueueUtils,
|
||||
lockHandler,
|
||||
gcsBufferSize,
|
||||
bucket,
|
||||
lockTimeout,
|
||||
stagingKeyBytes,
|
||||
validationMode);
|
||||
validationMode,
|
||||
gcsUtils);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user