mirror of
https://github.com/google/nomulus
synced 2026-05-19 14:21:48 +00:00
Compare commits
104 Commits
nomulus-20
...
nomulus-20
| Author | SHA1 | Date | |
|---|---|---|---|
|
f1dcb1299f | ||
|
d46594c610 | ||
|
|
0a9fa8cf23 | ||
|
db4bf90538 | ||
|
d6127e4c0c | ||
|
|
447bfa162b | ||
|
|
c9efa61198 | ||
|
|
054c0625a8 | ||
|
|
b03639d7fc | ||
|
|
bd9af0de84 | ||
|
|
ae911a5280 | ||
|
|
d57597f40f | ||
|
|
2641d0d462 | ||
|
|
5b41f0b9b6 | ||
|
|
1a26677d72 | ||
|
|
f1beeb4016 | ||
|
|
5c33286056 | ||
|
|
603a95d719 | ||
|
|
0a3774d3f7 | ||
|
|
cc60b27dd3 | ||
|
|
52c18f9967 | ||
|
|
5339b3cb6c | ||
|
|
d18dab3327 | ||
|
|
61932c1809 | ||
|
|
8eb8c810e8 | ||
|
c03a7b0b33 | ||
|
|
7a4c109b36 | ||
|
22b1b8d21a | ||
|
|
5bbabadafd | ||
|
|
6c73161ff8 | ||
|
|
7faee04422 | ||
|
|
b340b2b5e9 | ||
|
|
7f733cd16d | ||
|
|
60469479a4 | ||
|
|
5158673f21 | ||
|
|
743ca4106c | ||
|
|
2b99ee61d4 | ||
|
|
28a1cc613c | ||
|
|
9811cdb85c | ||
|
|
761ae612fd | ||
|
|
e2fa60a9c6 | ||
|
|
b04dfbf740 | ||
|
|
a1668ceafd | ||
|
|
406d49ac99 | ||
|
|
f7b82bc190 | ||
|
|
45c398149b | ||
|
|
183c0653fb | ||
|
|
fb002953c8 | ||
|
|
a369e57e5c | ||
|
|
f32d80fb9d | ||
|
|
afa5a353f1 | ||
|
|
c4c5ac85da | ||
|
|
4d0078607f | ||
|
|
2b78433682 | ||
|
|
a0fcd02ed2 | ||
|
|
58e413af89 | ||
|
|
38c8e81690 | ||
|
|
3beb207fcc | ||
|
|
8cf88b7e18 | ||
|
|
6ec2e9501d | ||
|
|
6849bf6914 | ||
|
|
34f3823960 | ||
|
|
bb5d2dcf0a | ||
|
|
6ce0211537 | ||
|
|
676616a172 | ||
|
|
62c556cebf | ||
|
|
535f84a912 | ||
|
|
d283cf1c90 | ||
|
|
f5d344d5c9 | ||
|
|
61d029d955 | ||
|
|
2195ba90fa | ||
|
|
e5b9ff1498 | ||
|
|
3f9fec98d5 | ||
|
|
4e30d020ca | ||
|
|
047444831b | ||
|
|
7adcbee5ad | ||
|
78a750b7e1 | ||
|
|
2e8a1c422d | ||
|
|
0e5605b175 | ||
|
|
a10b5d8b30 | ||
|
|
b7ce08dfdc | ||
|
|
a3e8bf219f | ||
|
|
546eba68bd | ||
|
|
81fcdbdcea | ||
|
|
2b91e3bb89 | ||
|
|
ce03556683 | ||
|
|
967304588b | ||
|
|
a2754a0eff | ||
|
|
276bbc09c2 | ||
|
|
fd461a78e7 | ||
|
|
0374ad60d8 | ||
|
|
fcc027e0c8 | ||
|
|
c3a4887845 | ||
|
|
a0b6437f4c | ||
|
|
a7210a26b4 | ||
|
|
c7096a1b71 | ||
|
|
30634ff404 | ||
|
|
4f71d780ab | ||
|
|
14ad56a392 | ||
|
|
a1b56b0521 | ||
|
|
3f41f7f444 | ||
|
|
4f6bcea63f | ||
|
|
bd0ef626a1 | ||
|
|
68304133c4 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -112,3 +112,7 @@ core/**/registrar_bin*.js
|
||||
core/**/registrar_dbg*.js
|
||||
core/**/registrar_bin*.css
|
||||
core/**/registrar_dbg*.css
|
||||
|
||||
# Appengine generated files
|
||||
core/WEB-INF/appengine-generated/*.bin
|
||||
core/WEB-INF/appengine-generated/*.xml
|
||||
|
||||
4
SECURITY.md
Normal file
4
SECURITY.md
Normal file
@@ -0,0 +1,4 @@
|
||||
To report a security issue, please use http://g.co/vulnz. We use
|
||||
http://g.co/vulnz for our intake, and do coordination and disclosure here on
|
||||
GitHub (including using GitHub Security Advisory). The Google Security Team will
|
||||
respond within 5 working days of your report on g.co/vulnz.
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
@@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
||||
@@ -100,6 +100,15 @@ public class DateTimeUtils {
|
||||
return ZonedDateTime.ofInstant(instant, ZoneId.of(dateTime.getZone().getID()).normalized());
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a Joda {@link DateTime} object to an equivalent java.time {@link ZonedDateTime}
|
||||
* object.
|
||||
*/
|
||||
public static ZonedDateTime toZonedDateTime(DateTime dateTime, ZoneId zoneId) {
|
||||
java.time.Instant instant = java.time.Instant.ofEpochMilli(dateTime.getMillis());
|
||||
return ZonedDateTime.ofInstant(instant, zoneId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a java.time {@link ZonedDateTime} object to an equivalent Joda {@link DateTime}
|
||||
* object.
|
||||
|
||||
@@ -79,8 +79,8 @@ PRESUBMITS = {
|
||||
r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
|
||||
("java", "js", "soy", "sql", "py", "sh", "gradle"), {
|
||||
".git", "/build/", "/generated/", "/generated_tests/",
|
||||
"node_modules/", "JUnitBackports.java", "registrar_bin.",
|
||||
"registrar_dbg.", "google-java-format-diff.py",
|
||||
"node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java",
|
||||
"registrar_bin.", "registrar_dbg.", "google-java-format-diff.py",
|
||||
"nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
|
||||
}, REQUIRED):
|
||||
"File did not include the license header.",
|
||||
@@ -202,6 +202,8 @@ PRESUBMITS = {
|
||||
"java",
|
||||
# ActivityReportingQueryBuilder deals with Dremel queries
|
||||
{"src/test", "ActivityReportingQueryBuilder.java",
|
||||
# This class contains helper method to make queries in Beam.
|
||||
"RegistryJpaIO.java",
|
||||
# TODO(b/179158393): Remove everything below, which should be done
|
||||
# using Criteria
|
||||
"ForeignKeyIndex.java",
|
||||
@@ -213,6 +215,7 @@ PRESUBMITS = {
|
||||
"RdapDomainSearchAction.java",
|
||||
"RdapNameserverSearchAction.java",
|
||||
"RdapSearchActionBase.java",
|
||||
"ReadOnlyCheckingEntityManager.java",
|
||||
"RegistryQuery",
|
||||
},
|
||||
):
|
||||
|
||||
Binary file not shown.
@@ -44,7 +44,6 @@ def outcastTestPatterns = [
|
||||
"google/registry/flows/domain/DomainCreateFlowTest.*",
|
||||
"google/registry/flows/domain/DomainUpdateFlowTest.*",
|
||||
"google/registry/tools/CreateDomainCommandTest.*",
|
||||
"google/registry/tools/server/CreatePremiumListActionTest.*",
|
||||
]
|
||||
|
||||
// Tests that fail when running Gradle in a docker container, e. g. when
|
||||
@@ -70,15 +69,14 @@ def dockerIncompatibleTestPatterns = [
|
||||
// Nomulus classes, e.g., threads and objects retained by frameworks.
|
||||
// TODO(weiminyu): identify cause and fix offending tests.
|
||||
def fragileTestPatterns = [
|
||||
// Problem seems to lie with AppEngine TaskQueue for test.
|
||||
"google/registry/cron/TldFanoutActionTest.*",
|
||||
// Test Datastore inexplicably aborts transaction.
|
||||
"google/registry/model/tmch/ClaimsListShardTest.*",
|
||||
// Creates large object (64MBytes), occasionally throws OOM error.
|
||||
"google/registry/model/server/KmsSecretRevisionTest.*",
|
||||
// Changes cache timeouts and for some reason appears to have contention
|
||||
// with other tests.
|
||||
"google/registry/whois/WhoisCommandFactoryTest.*",
|
||||
// Currently changes a global configuration parameter that for some reason
|
||||
// results in timestamp inversions for other tests. TODO(mmuller): fix.
|
||||
"google/registry/flows/host/HostInfoFlowTest.*",
|
||||
] + dockerIncompatibleTestPatterns
|
||||
|
||||
sourceSets {
|
||||
@@ -184,6 +182,7 @@ dependencies {
|
||||
compile deps['com.google.monitoring-client:metrics']
|
||||
compile deps['com.google.monitoring-client:stackdriver']
|
||||
compile deps['com.google.api-client:google-api-client-java6']
|
||||
compile deps['com.google.api.grpc:proto-google-cloud-tasks-v2']
|
||||
compile deps['com.google.apis:google-api-services-admin-directory']
|
||||
compile deps['com.google.apis:google-api-services-appengine']
|
||||
compile deps['com.google.apis:google-api-services-bigquery']
|
||||
@@ -194,6 +193,7 @@ dependencies {
|
||||
compile deps['com.google.apis:google-api-services-groupssettings']
|
||||
compile deps['com.google.apis:google-api-services-monitoring']
|
||||
compile deps['com.google.apis:google-api-services-sheets']
|
||||
compile deps['com.google.apis:google-api-services-storage']
|
||||
testCompile deps['com.google.appengine:appengine-api-stubs']
|
||||
compile deps['com.google.appengine.tools:appengine-gcs-client']
|
||||
compile deps['com.google.appengine.tools:appengine-mapreduce']
|
||||
@@ -215,9 +215,13 @@ dependencies {
|
||||
compile deps['com.google.flogger:flogger']
|
||||
runtime deps['com.google.flogger:flogger-system-backend']
|
||||
compile deps['com.google.guava:guava']
|
||||
compile deps['com.google.protobuf:protobuf-java']
|
||||
gradleLint.ignore('unused-dependency') {
|
||||
compile deps['com.google.gwt:gwt-user']
|
||||
}
|
||||
compile deps['com.google.cloud:google-cloud-core']
|
||||
compile deps['com.google.cloud:google-cloud-storage']
|
||||
compile deps['com.google.cloud:google-cloud-tasks']
|
||||
compile deps['com.google.http-client:google-http-client']
|
||||
compile deps['com.google.http-client:google-http-client-appengine']
|
||||
compile deps['com.google.http-client:google-http-client-jackson2']
|
||||
@@ -312,6 +316,7 @@ dependencies {
|
||||
annotationProcessor project(':processor')
|
||||
testAnnotationProcessor project(':processor')
|
||||
|
||||
testCompile deps['com.google.cloud:google-cloud-nio']
|
||||
testCompile deps['com.google.appengine:appengine-testing']
|
||||
testCompile deps['com.google.guava:guava-testlib']
|
||||
testCompile deps['com.google.monitoring-client:contrib']
|
||||
@@ -777,6 +782,10 @@ if (environment in ['alpha', 'crash']) {
|
||||
mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
|
||||
metaData: 'google/registry/beam/invoicing_pipeline_metadata.json'
|
||||
],
|
||||
[
|
||||
mainClass: 'google.registry.beam.rde.RdePipeline',
|
||||
metaData: 'google/registry/beam/rde_pipeline_metadata.json'
|
||||
],
|
||||
]
|
||||
project.tasks.create("stage_beam_pipelines") {
|
||||
doLast {
|
||||
|
||||
@@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33
|
||||
com.google.dagger:dagger-spi:2.33
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotation:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_check_api:2.3.4
|
||||
com.google.errorprone:error_prone_core:2.3.4
|
||||
com.google.errorprone:error_prone_type_annotations:2.3.4
|
||||
com.google.errorprone:javac-shaded:9-dev-r4023-3
|
||||
com.google.googlejavaformat:google-java-format:1.5
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.protobuf:protobuf-java:3.4.0
|
||||
@@ -32,7 +32,7 @@ javax.inject:javax.inject:1
|
||||
javax.persistence:javax.persistence-api:2.2
|
||||
net.ltgt.gradle.incap:incap:0.2
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.checkerframework:dataflow:3.0.0
|
||||
org.checkerframework:javacutil:3.0.0
|
||||
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
|
||||
|
||||
@@ -54,12 +54,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -76,17 +79,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -97,31 +100,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -133,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -150,17 +155,17 @@ commons-logging:commons-logging:1.2
|
||||
dnsjava:dnsjava:3.3.1
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -221,7 +226,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
@@ -258,7 +263,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -53,12 +53,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -75,17 +78,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -96,30 +99,32 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -131,8 +136,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -148,17 +153,17 @@ commons-logging:commons-logging:1.2
|
||||
dnsjava:dnsjava:3.3.1
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -215,7 +220,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
@@ -251,7 +256,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -102,31 +105,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -138,8 +143,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -158,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
|
||||
guru.nidi:graphviz-java:0.17.0
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -232,7 +237,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
@@ -272,7 +277,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -102,31 +105,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -138,8 +143,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -158,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
|
||||
guru.nidi:graphviz-java:0.17.0
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -231,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
@@ -271,7 +276,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -54,12 +54,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -76,17 +79,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -97,31 +100,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -133,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -150,17 +155,17 @@ commons-logging:commons-logging:1.2
|
||||
dnsjava:dnsjava:3.3.1
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -221,7 +226,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
@@ -258,7 +263,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -53,12 +53,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -75,17 +78,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -96,30 +99,32 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -131,8 +136,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -148,17 +153,17 @@ commons-logging:commons-logging:1.2
|
||||
dnsjava:dnsjava:3.3.1
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -216,7 +221,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
@@ -252,7 +257,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -101,31 +104,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -137,8 +142,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -157,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
|
||||
guru.nidi:graphviz-java:0.17.0
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -231,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
@@ -270,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -101,31 +104,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -137,8 +142,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -157,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
|
||||
guru.nidi:graphviz-java:0.17.0
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -231,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
@@ -270,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -101,31 +104,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -137,8 +142,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -157,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
|
||||
guru.nidi:graphviz-java:0.17.0
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -231,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
@@ -270,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -102,31 +105,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -138,8 +143,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
@@ -158,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
|
||||
guru.nidi:graphviz-java:0.17.0
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.65
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -231,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
||||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
@@ -271,7 +276,7 @@ org.testcontainers:database-commons:1.15.2
|
||||
org.testcontainers:jdbc:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33
|
||||
com.google.dagger:dagger-spi:2.33
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotation:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_check_api:2.3.4
|
||||
com.google.errorprone:error_prone_core:2.3.4
|
||||
com.google.errorprone:error_prone_type_annotations:2.3.4
|
||||
com.google.errorprone:javac-shaded:9-dev-r4023-3
|
||||
com.google.googlejavaformat:google-java-format:1.5
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.protobuf:protobuf-java:3.4.0
|
||||
@@ -30,7 +30,7 @@ javax.inject:javax.inject:1
|
||||
javax.persistence:javax.persistence-api:2.2
|
||||
net.ltgt.gradle.incap:incap:0.2
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.checkerframework:dataflow:3.0.0
|
||||
org.checkerframework:javacutil:3.0.0
|
||||
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
|
||||
|
||||
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -54,12 +54,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -76,7 +79,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -84,10 +87,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -98,32 +101,35 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -135,9 +141,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -158,17 +164,17 @@ commons-logging:commons-logging:1.2
|
||||
dnsjava:dnsjava:3.3.1
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.102
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -199,7 +205,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
net.java.dev.jna:jna:5.5.0
|
||||
@@ -307,7 +313,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -53,12 +53,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -75,7 +78,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -83,10 +86,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -97,31 +100,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -133,9 +139,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -156,17 +162,17 @@ commons-logging:commons-logging:1.2
|
||||
dnsjava:dnsjava:3.3.1
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.102
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -194,7 +200,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
net.java.dev.jna:jna:5.5.0
|
||||
@@ -301,7 +307,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
||||
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,7 +83,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -88,10 +91,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -103,32 +106,35 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -140,9 +146,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -167,17 +173,17 @@ guru.nidi:graphviz-java:0.17.0
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.102
|
||||
io.github.java-diff-utils:java-diff-utils:4.9
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -208,7 +214,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.arnx:nashorn-promise:0.1.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -320,7 +326,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
|
||||
com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.66.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,7 +83,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -88,10 +91,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
||||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
@@ -103,32 +106,35 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.cloud:google-cloud-tasks:1.33.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -140,9 +146,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -167,17 +173,17 @@ guru.nidi:graphviz-java:0.17.0
|
||||
io.dropwizard.metrics:metrics-core:3.2.6
|
||||
io.github.classgraph:classgraph:4.8.102
|
||||
io.github.java-diff-utils:java-diff-utils:4.9
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
io.grpc:grpc-alts:1.39.0
|
||||
io.grpc:grpc-api:1.39.0
|
||||
io.grpc:grpc-auth:1.39.0
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.39.0
|
||||
io.grpc:grpc-grpclb:1.39.0
|
||||
io.grpc:grpc-netty-shaded:1.39.0
|
||||
io.grpc:grpc-netty:1.32.2
|
||||
io.grpc:grpc-protobuf-lite:1.36.0
|
||||
io.grpc:grpc-protobuf:1.36.0
|
||||
io.grpc:grpc-stub:1.36.0
|
||||
io.grpc:grpc-protobuf-lite:1.39.0
|
||||
io.grpc:grpc-protobuf:1.39.0
|
||||
io.grpc:grpc-stub:1.39.0
|
||||
io.netty:netty-buffer:4.1.51.Final
|
||||
io.netty:netty-codec-http2:4.1.51.Final
|
||||
io.netty:netty-codec-http:4.1.51.Final
|
||||
@@ -208,7 +214,7 @@ javax.validation:validation-api:1.0.0.GA
|
||||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.arnx:nashorn-promise:0.1.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -321,7 +327,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
||||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
||||
@@ -18,8 +18,10 @@ import static com.google.appengine.api.ThreadManager.currentRequestThreadFactory
|
||||
import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.BUCKET_OVERRIDE_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.FROM_TIME_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.TO_TIME_PARAM;
|
||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static java.util.concurrent.Executors.newFixedThreadPool;
|
||||
@@ -32,6 +34,9 @@ import google.registry.cron.CommitLogFanoutAction;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.Parameter;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.inject.Qualifier;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -75,6 +80,12 @@ public final class BackupModule {
|
||||
return extractRequiredDatetimeParameter(req, UPPER_CHECKPOINT_TIME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(BUCKET_OVERRIDE_PARAM)
|
||||
static Optional<String> provideBucketOverride(HttpServletRequest req) {
|
||||
return extractOptionalParameter(req, BUCKET_OVERRIDE_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(FROM_TIME_PARAM)
|
||||
static DateTime provideFromTime(HttpServletRequest req) {
|
||||
@@ -92,4 +103,9 @@ public final class BackupModule {
|
||||
static ListeningExecutorService provideListeningExecutorService() {
|
||||
return listeningDecorator(newFixedThreadPool(NUM_THREADS, currentRequestThreadFactory()));
|
||||
}
|
||||
|
||||
@Provides
|
||||
static ScheduledExecutorService provideScheduledExecutorService() {
|
||||
return Executors.newSingleThreadScheduledExecutor();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,12 +56,16 @@ public class BackupUtils {
|
||||
*
|
||||
* <p>The iterator reads from the stream on demand, and as such will fail if the stream is closed.
|
||||
*/
|
||||
public static Iterator<ImmutableObject> createDeserializingIterator(final InputStream input) {
|
||||
public static Iterator<ImmutableObject> createDeserializingIterator(
|
||||
final InputStream input, boolean withAppIdOverride) {
|
||||
return new AbstractIterator<ImmutableObject>() {
|
||||
@Override
|
||||
protected ImmutableObject computeNext() {
|
||||
EntityProto proto = new EntityProto();
|
||||
if (proto.parseDelimitedFrom(input)) { // False means end of stream; other errors throw.
|
||||
if (withAppIdOverride) {
|
||||
proto = EntityImports.fixEntity(proto);
|
||||
}
|
||||
return auditedOfy().load().fromEntity(EntityTranslator.createFromPb(proto));
|
||||
}
|
||||
return endOfData();
|
||||
@@ -70,6 +74,7 @@ public class BackupUtils {
|
||||
}
|
||||
|
||||
public static ImmutableList<ImmutableObject> deserializeEntities(byte[] bytes) {
|
||||
return ImmutableList.copyOf(createDeserializingIterator(new ByteArrayInputStream(bytes)));
|
||||
return ImmutableList.copyOf(
|
||||
createDeserializingIterator(new ByteArrayInputStream(bytes), false));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ public final class CommitLogImports {
|
||||
InputStream inputStream) {
|
||||
try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
|
||||
InputStream input = new BufferedInputStream(inputStream)) {
|
||||
Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
|
||||
Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
|
||||
checkState(commitLogs.hasNext());
|
||||
checkState(commitLogs.next() instanceof CommitLogCheckpoint);
|
||||
|
||||
|
||||
@@ -66,6 +66,8 @@ import org.joda.time.Duration;
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/deleteOldCommitLogs",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
// No longer needed in SQL. Subject to future removal.
|
||||
@Deprecated
|
||||
public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
|
||||
private static final int NUM_MAP_SHARDS = 20;
|
||||
|
||||
115
core/src/main/java/google/registry/backup/EntityImports.java
Normal file
115
core/src/main/java/google/registry/backup/EntityImports.java
Normal file
@@ -0,0 +1,115 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import com.google.apphosting.api.ApiProxy;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.Path;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Objects;
|
||||
|
||||
/** Utilities for handling imported Datastore entities. */
|
||||
public class EntityImports {
|
||||
|
||||
/**
|
||||
* Transitively sets the {@code appId} of all keys in a foreign entity to that of the current
|
||||
* system.
|
||||
*/
|
||||
public static EntityProto fixEntity(EntityProto entityProto) {
|
||||
String currentAappId = ApiProxy.getCurrentEnvironment().getAppId();
|
||||
if (Objects.equals(currentAappId, entityProto.getKey().getApp())) {
|
||||
return entityProto;
|
||||
}
|
||||
return fixEntity(entityProto, currentAappId);
|
||||
}
|
||||
|
||||
private static EntityProto fixEntity(EntityProto entityProto, String appId) {
|
||||
if (entityProto.hasKey()) {
|
||||
fixKey(entityProto, appId);
|
||||
}
|
||||
|
||||
for (OnestoreEntity.Property property : entityProto.mutablePropertys()) {
|
||||
fixProperty(property, appId);
|
||||
}
|
||||
|
||||
for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
|
||||
fixProperty(property, appId);
|
||||
}
|
||||
|
||||
// CommitLogMutation embeds an entity as bytes, which needs additional fixes.
|
||||
if (isCommitLogMutation(entityProto)) {
|
||||
fixMutationEntityProtoBytes(entityProto, appId);
|
||||
}
|
||||
return entityProto;
|
||||
}
|
||||
|
||||
private static boolean isCommitLogMutation(EntityProto entityProto) {
|
||||
if (!entityProto.hasKey()) {
|
||||
return false;
|
||||
}
|
||||
Path path = entityProto.getKey().getPath();
|
||||
if (path.elementSize() == 0) {
|
||||
return false;
|
||||
}
|
||||
return Objects.equals(
|
||||
path.getElement(path.elementSize() - 1).getType(StandardCharsets.UTF_8),
|
||||
"CommitLogMutation");
|
||||
}
|
||||
|
||||
private static void fixMutationEntityProtoBytes(EntityProto entityProto, String appId) {
|
||||
for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
|
||||
if (Objects.equals(property.getName(), "entityProtoBytes")) {
|
||||
OnestoreEntity.PropertyValue value = property.getValue();
|
||||
EntityProto fixedProto =
|
||||
fixEntity(bytesToEntityProto(value.getStringValueAsBytes()), appId);
|
||||
value.setStringValueAsBytes(fixedProto.toByteArray());
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void fixKey(EntityProto entityProto, String appId) {
|
||||
entityProto.getMutableKey().setApp(appId);
|
||||
}
|
||||
|
||||
private static void fixKey(ReferenceValue referenceValue, String appId) {
|
||||
referenceValue.setApp(appId);
|
||||
}
|
||||
|
||||
private static void fixProperty(OnestoreEntity.Property property, String appId) {
|
||||
OnestoreEntity.PropertyValue value = property.getMutableValue();
|
||||
if (value.hasReferenceValue()) {
|
||||
fixKey(value.getMutableReferenceValue(), appId);
|
||||
return;
|
||||
}
|
||||
if (property.getMeaningEnum().equals(Meaning.ENTITY_PROTO)) {
|
||||
EntityProto embeddedProto = bytesToEntityProto(value.getStringValueAsBytes());
|
||||
fixEntity(embeddedProto, appId);
|
||||
value.setStringValueAsBytes(embeddedProto.toByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
private static EntityProto bytesToEntityProto(byte[] bytes) {
|
||||
EntityProto entityProto = new EntityProto();
|
||||
boolean isParsed = entityProto.parseFrom(bytes);
|
||||
if (!isParsed) {
|
||||
throw new IllegalStateException("Failed to parse raw bytes as EntityProto.");
|
||||
}
|
||||
return entityProto;
|
||||
}
|
||||
}
|
||||
@@ -28,18 +28,17 @@ import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static java.nio.channels.Channels.newOutputStream;
|
||||
import static java.util.Comparator.comparingLong;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
@@ -74,7 +73,8 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
|
||||
public static final String DIFF_FILE_PREFIX = "commit_diff_until_";
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
@Inject @Config("commitLogDiffExportBatchSize") int batchSize;
|
||||
@Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime;
|
||||
@@ -102,13 +102,13 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
|
||||
logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
|
||||
// Open an output channel to GCS, wrapped in a stream for convenience.
|
||||
try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
|
||||
new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString())
|
||||
.addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString())
|
||||
.addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size()))
|
||||
.build()))) {
|
||||
try (OutputStream gcsStream =
|
||||
gcsUtils.openOutputStream(
|
||||
BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
ImmutableMap.of(
|
||||
LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(),
|
||||
UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(),
|
||||
NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) {
|
||||
// Export the upper checkpoint itself.
|
||||
serializeEntity(upperCheckpoint, gcsStream);
|
||||
// If there are no manifests to export, stop early, now that we've written out the file with
|
||||
|
||||
@@ -15,30 +15,32 @@
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static google.registry.util.DateTimeUtils.latestOf;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.common.util.concurrent.ListeningExecutorService;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import google.registry.backup.BackupModule.Backups;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Provider;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Utility class to list commit logs diff files stored on GCS. */
|
||||
@@ -46,33 +48,49 @@ class GcsDiffFileLister {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
@Inject @Backups ListeningExecutorService executor;
|
||||
@Inject GcsDiffFileLister() {}
|
||||
/** Timeout for retrieving per-file information from GCS. */
|
||||
private static final Duration FILE_INFO_TIMEOUT_DURATION = Duration.ofMinutes(1);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Backups Provider<ListeningExecutorService> executorProvider;
|
||||
@Inject ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
@Inject
|
||||
GcsDiffFileLister() {}
|
||||
|
||||
/**
|
||||
* Traverses the sequence of diff files backwards from checkpointTime and inserts the file
|
||||
* metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
|
||||
* metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
|
||||
* more files are missing.
|
||||
*
|
||||
* @throws UncheckedExecutionException wrapping a {@link java.util.concurrent.TimeoutException} if
|
||||
* the GCS call fails to finish within one minute, or wrapping any other exception if
|
||||
* something else goes wrong.
|
||||
*/
|
||||
private boolean constructDiffSequence(
|
||||
Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
|
||||
String gcsBucket,
|
||||
Map<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo,
|
||||
DateTime fromTime,
|
||||
DateTime lastTime,
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence) {
|
||||
TreeMap<DateTime, BlobInfo> sequence) {
|
||||
DateTime checkpointTime = lastTime;
|
||||
while (isBeforeOrAt(fromTime, checkpointTime)) {
|
||||
GcsFileMetadata metadata;
|
||||
if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
|
||||
metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
|
||||
BlobInfo blobInfo;
|
||||
if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) {
|
||||
blobInfo =
|
||||
Futures.getUnchecked(
|
||||
Futures.withTimeout(
|
||||
upperBoundTimesToBlobInfo.get(checkpointTime),
|
||||
FILE_INFO_TIMEOUT_DURATION,
|
||||
scheduledExecutorService));
|
||||
} else {
|
||||
String filename = DIFF_FILE_PREFIX + checkpointTime;
|
||||
logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
|
||||
metadata = getMetadata(filename);
|
||||
blobInfo = getBlobInfo(gcsBucket, filename);
|
||||
|
||||
// If we hit a gap, quit.
|
||||
if (metadata == null) {
|
||||
if (blobInfo == null) {
|
||||
logger.atInfo().log(
|
||||
"Gap discovered in sequence terminating at %s, missing file: %s",
|
||||
sequence.lastKey(), filename);
|
||||
@@ -80,14 +98,15 @@ class GcsDiffFileLister {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sequence.put(checkpointTime, metadata);
|
||||
checkpointTime = getLowerBoundTime(metadata);
|
||||
sequence.put(checkpointTime, blobInfo);
|
||||
checkpointTime = getLowerBoundTime(blobInfo);
|
||||
}
|
||||
logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
|
||||
return true;
|
||||
}
|
||||
|
||||
ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
|
||||
ImmutableList<BlobInfo> listDiffFiles(
|
||||
String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) {
|
||||
logger.atInfo().log("Requested restore from time: %s", fromTime);
|
||||
if (toTime != null) {
|
||||
logger.atInfo().log(" Until time: %s", toTime);
|
||||
@@ -95,66 +114,74 @@ class GcsDiffFileLister {
|
||||
// List all of the diff files on GCS and build a map from each file's upper checkpoint time
|
||||
// (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
|
||||
// an upper checkpoint time > fromTime.
|
||||
TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
|
||||
= new TreeMap<>();
|
||||
Iterator<ListItem> listItems;
|
||||
TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
|
||||
String commitLogDiffPrefix = getCommitLogDiffPrefix(fromTime, toTime);
|
||||
ImmutableList<String> filenames;
|
||||
try {
|
||||
// TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
|
||||
listItems = gcsService.list(
|
||||
gcsBucket,
|
||||
new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
|
||||
filenames =
|
||||
gcsUtils.listFolderObjects(gcsBucket, commitLogDiffPrefix).stream()
|
||||
.map(s -> commitLogDiffPrefix + s)
|
||||
.collect(toImmutableList());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
DateTime lastUpperBoundTime = START_OF_TIME;
|
||||
while (listItems.hasNext()) {
|
||||
final String filename = listItems.next().getName();
|
||||
DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToMetadata.put(upperBoundTime, executor.submit(() -> getMetadata(filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
if (upperBoundTimesToMetadata.isEmpty()) {
|
||||
logger.atInfo().log("No files found");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
// Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
|
||||
// last file that we found) and finding its previous file until we either run out of files or
|
||||
// get to one that precedes "fromTime".
|
||||
//
|
||||
// GCS file listing is eventually consistent, so it's possible that we are missing a file. The
|
||||
// metadata of a file is sufficient to identify the preceding file, so if we start from the
|
||||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet = !constructDiffSequence(
|
||||
upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
// Verify that all of the elements in the original set are represented in the sequence. If we
|
||||
// find anything that's not represented, construct a sequence for it.
|
||||
boolean checkForMoreExtraDiffs = true; // Always loop at least once.
|
||||
while (checkForMoreExtraDiffs) {
|
||||
checkForMoreExtraDiffs = false;
|
||||
for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
|
||||
if (!isInRange(key, fromTime, toTime)) {
|
||||
break;
|
||||
}
|
||||
if (!sequence.containsKey(key)) {
|
||||
constructDiffSequence(upperBoundTimesToMetadata, fromTime, key, sequence);
|
||||
checkForMoreExtraDiffs = true;
|
||||
inconsistentFileSet = true;
|
||||
break;
|
||||
TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
|
||||
ListeningExecutorService executor = executorProvider.get();
|
||||
try {
|
||||
for (String filename : filenames) {
|
||||
String strippedFilename = filename.replaceFirst(DIFF_FILE_PREFIX, "");
|
||||
DateTime upperBoundTime = DateTime.parse(strippedFilename);
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToBlobInfo.put(
|
||||
upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (upperBoundTimesToBlobInfo.isEmpty()) {
|
||||
logger.atInfo().log("No files found");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
checkState(
|
||||
!inconsistentFileSet,
|
||||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
// Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e.
|
||||
// the last file that we found) and finding its previous file until we either run out of files
|
||||
// or get to one that precedes "fromTime".
|
||||
//
|
||||
// GCS file listing is eventually consistent, so it's possible that we are missing a file. The
|
||||
// metadata of a file is sufficient to identify the preceding file, so if we start from the
|
||||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet =
|
||||
!constructDiffSequence(
|
||||
gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
// Verify that all of the elements in the original set are represented in the sequence. If we
|
||||
// find anything that's not represented, construct a sequence for it.
|
||||
boolean checkForMoreExtraDiffs = true; // Always loop at least once.
|
||||
while (checkForMoreExtraDiffs) {
|
||||
checkForMoreExtraDiffs = false;
|
||||
for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) {
|
||||
if (!isInRange(key, fromTime, toTime)) {
|
||||
break;
|
||||
}
|
||||
if (!sequence.containsKey(key)) {
|
||||
constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
|
||||
checkForMoreExtraDiffs = true;
|
||||
inconsistentFileSet = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checkState(
|
||||
!inconsistentFileSet,
|
||||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
} finally {
|
||||
executor.shutdown();
|
||||
}
|
||||
|
||||
logger.atInfo().log(
|
||||
"Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
|
||||
@@ -171,15 +198,43 @@ class GcsDiffFileLister {
|
||||
return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end));
|
||||
}
|
||||
|
||||
private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
|
||||
return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
private DateTime getLowerBoundTime(BlobInfo blobInfo) {
|
||||
return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
}
|
||||
|
||||
private GcsFileMetadata getMetadata(String filename) {
|
||||
try {
|
||||
return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
private BlobInfo getBlobInfo(String gcsBucket, String filename) {
|
||||
return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a prefix guaranteed to cover all commit log diff files in the given range.
|
||||
*
|
||||
* <p>The listObjects call can be fairly slow if we search over many thousands or tens of
|
||||
* thousands of files, so we restrict the search space. The commit logs have a file format of
|
||||
* "commit_diff_until_2021-05-11T06:48:00.070Z" so we can often filter down as far as the hour.
|
||||
*
|
||||
* <p>Here, we get the longest prefix possible based on which fields (year, month, day, hour) the
|
||||
* times in question have in common.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static String getCommitLogDiffPrefix(DateTime from, @Nullable DateTime to) {
|
||||
StringBuilder result = new StringBuilder(DIFF_FILE_PREFIX);
|
||||
if (to == null || from.getYear() != to.getYear()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(from.getYear()).append('-');
|
||||
if (from.getMonthOfYear() != to.getMonthOfYear()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02d-", from.getMonthOfYear()));
|
||||
if (from.getDayOfMonth() != to.getDayOfMonth()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02dT", from.getDayOfMonth()));
|
||||
if (from.getHourOfDay() != to.getHourOfDay()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02d:", from.getHourOfDay()));
|
||||
return result.toString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,72 +14,92 @@
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.DRY_RUN_PARAM;
|
||||
import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.joda.time.Duration.standardHours;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.Key;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
|
||||
import google.registry.model.replay.DatastoreEntity;
|
||||
import google.registry.model.replay.DatastoreOnlyEntity;
|
||||
import google.registry.model.replay.NonReplicatedEntity;
|
||||
import google.registry.model.replay.ReplaySpecializer;
|
||||
import google.registry.model.replay.SqlReplayCheckpoint;
|
||||
import google.registry.model.server.Lock;
|
||||
import google.registry.model.translators.VKeyTranslatorFactory;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Method;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.schema.replay.DatastoreEntity;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
import google.registry.schema.replay.NonReplicatedEntity;
|
||||
import google.registry.schema.replay.ReplaySpecializer;
|
||||
import google.registry.schema.replay.SqlReplayCheckpoint;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.RequestStatusChecker;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.Seconds;
|
||||
|
||||
/** Action that replays commit logs to Cloud SQL to keep it up to date. */
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = ReplayCommitLogsToSqlAction.PATH,
|
||||
method = Action.Method.POST,
|
||||
method = Method.POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
|
||||
static final String PATH = "/_dr/task/replayCommitLogsToSql";
|
||||
|
||||
private static final int BLOCK_SIZE =
|
||||
1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final Duration LEASE_LENGTH = standardHours(1);
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
private static final Duration LEASE_LENGTH = standardHours(1);
|
||||
// Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
|
||||
// request timeouts
|
||||
private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject Response response;
|
||||
@Inject RequestStatusChecker requestStatusChecker;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
@Inject Clock clock;
|
||||
|
||||
@Inject
|
||||
@Config("commitLogGcsBucket")
|
||||
String gcsBucket;
|
||||
|
||||
/** If true, will exit after logging the commit log files that would otherwise be replayed. */
|
||||
@Inject
|
||||
@Parameter(DRY_RUN_PARAM)
|
||||
boolean dryRun;
|
||||
|
||||
@Inject
|
||||
ReplayCommitLogsToSqlAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc());
|
||||
DateTime startTime = clock.nowUtc();
|
||||
MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(startTime);
|
||||
if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
|
||||
String message =
|
||||
String.format(
|
||||
@@ -103,44 +123,117 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
replayFiles();
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
logger.atInfo().log("ReplayCommitLogsToSqlAction completed successfully.");
|
||||
logger.atInfo().log("Beginning replay of commit logs.");
|
||||
String resultMessage;
|
||||
if (dryRun) {
|
||||
resultMessage = executeDryRun();
|
||||
} else {
|
||||
resultMessage = replayFiles(startTime);
|
||||
}
|
||||
response.setStatus(SC_OK);
|
||||
response.setPayload(resultMessage);
|
||||
logger.atInfo().log(resultMessage);
|
||||
} catch (Throwable t) {
|
||||
String message = "Errored out replaying files.";
|
||||
logger.atSevere().withCause(t).log(message);
|
||||
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
|
||||
response.setPayload(message);
|
||||
} finally {
|
||||
lock.ifPresent(Lock::release);
|
||||
}
|
||||
}
|
||||
|
||||
private void replayFiles() {
|
||||
private String executeDryRun() {
|
||||
// Start at the first millisecond we haven't seen yet
|
||||
DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
// If there's an inconsistent file set, this will throw IllegalStateException and the job
|
||||
// will try later -- this is likely because an export hasn't finished yet.
|
||||
ImmutableList<GcsFileMetadata> commitLogFiles =
|
||||
diffLister.listDiffFiles(fromTime, /* current time */ null);
|
||||
for (GcsFileMetadata metadata : commitLogFiles) {
|
||||
// One transaction per GCS file
|
||||
jpaTm().transact(() -> processFile(metadata));
|
||||
}
|
||||
logger.atInfo().log("Replayed %d commit log files to SQL successfully.", commitLogFiles.size());
|
||||
DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
// Search through the end of the hour
|
||||
DateTime searchEndTime =
|
||||
searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
|
||||
ImmutableList<String> fileBatch =
|
||||
diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime).stream()
|
||||
.map(BlobInfo::getName)
|
||||
.collect(toImmutableList());
|
||||
return String.format(
|
||||
"Running in dry-run mode, the first set of commit log files processed would be from "
|
||||
+ "searching from %s to %s and would contain %d file(s). They are (limit 10): \n%s",
|
||||
searchStartTime,
|
||||
searchEndTime,
|
||||
fileBatch.size(),
|
||||
fileBatch.stream().limit(10).collect(toImmutableList()));
|
||||
}
|
||||
|
||||
private void processFile(GcsFileMetadata metadata) {
|
||||
try (InputStream input =
|
||||
Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
private String replayFiles(DateTime startTime) {
|
||||
DateTime replayTimeoutTime = startTime.plus(REPLAY_TIMEOUT_DURATION);
|
||||
DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
int filesProcessed = 0;
|
||||
int transactionsProcessed = 0;
|
||||
// Starting from one millisecond after the last file we processed, search for and import files
|
||||
// one hour at a time until we catch up to the current time or we hit the replay timeout (in
|
||||
// which case the next run will pick up from where we leave off).
|
||||
//
|
||||
// We use hour-long batches because GCS supports filename prefix-based searches.
|
||||
while (true) {
|
||||
if (isAtOrAfter(clock.nowUtc(), replayTimeoutTime)) {
|
||||
return createResponseString(
|
||||
"Reached max execution time", startTime, filesProcessed, transactionsProcessed);
|
||||
}
|
||||
if (isBeforeOrAt(clock.nowUtc(), searchStartTime)) {
|
||||
return createResponseString(
|
||||
"Caught up to current time", startTime, filesProcessed, transactionsProcessed);
|
||||
}
|
||||
// Search through the end of the hour
|
||||
DateTime searchEndTime =
|
||||
searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
|
||||
ImmutableList<BlobInfo> fileBatch =
|
||||
diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime);
|
||||
if (fileBatch.isEmpty()) {
|
||||
logger.atInfo().log(
|
||||
"No remaining files found in hour %s, continuing search in the next hour.",
|
||||
searchStartTime.toString("yyyy-MM-dd HH"));
|
||||
}
|
||||
for (BlobInfo file : fileBatch) {
|
||||
transactionsProcessed += processFile(file);
|
||||
filesProcessed++;
|
||||
if (clock.nowUtc().isAfter(replayTimeoutTime)) {
|
||||
return createResponseString(
|
||||
"Reached max execution time", startTime, filesProcessed, transactionsProcessed);
|
||||
}
|
||||
}
|
||||
searchStartTime = searchEndTime.plusMillis(1);
|
||||
}
|
||||
}
|
||||
|
||||
private String createResponseString(
|
||||
String msg, DateTime startTime, int filesProcessed, int transactionsProcessed) {
|
||||
double tps =
|
||||
(double) transactionsProcessed
|
||||
/ (double) Seconds.secondsBetween(startTime, clock.nowUtc()).getSeconds();
|
||||
return String.format(
|
||||
"%s after replaying %d file(s) containing %d total transaction(s) (%.2f tx/s).",
|
||||
msg, filesProcessed, transactionsProcessed, tps);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replays the commit logs in the given commit log file and returns the number of transactions
|
||||
* committed.
|
||||
*/
|
||||
private int processFile(BlobInfo metadata) {
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
// Load and process the Datastore transactions one at a time
|
||||
ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
|
||||
CommitLogImports.loadEntitiesByTransaction(input);
|
||||
allTransactions.forEach(this::replayTransaction);
|
||||
allTransactions.forEach(
|
||||
transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
|
||||
// if we succeeded, set the last-seen time
|
||||
DateTime checkpoint =
|
||||
DateTime.parse(
|
||||
metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
|
||||
SqlReplayCheckpoint.set(checkpoint);
|
||||
logger.atInfo().log("Replayed %d transactions from commit log file.", allTransactions.size());
|
||||
DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(checkpoint));
|
||||
logger.atInfo().log(
|
||||
"Replayed %d transactions from commit log file %s with size %d B.",
|
||||
allTransactions.size(), metadata.getName(), metadata.getSize());
|
||||
return allTransactions.size();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
throw new RuntimeException(
|
||||
"Errored out while replaying commit log file " + metadata.getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,20 +252,25 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
|
||||
private void handleEntityPut(Entity entity) {
|
||||
Object ofyPojo = auditedOfy().toPojo(entity);
|
||||
if (ofyPojo instanceof DatastoreEntity) {
|
||||
DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
|
||||
datastoreEntity
|
||||
.toSqlEntity()
|
||||
.ifPresent(
|
||||
sqlEntity -> {
|
||||
ReplaySpecializer.beforeSqlSave(sqlEntity);
|
||||
jpaTm().put(sqlEntity);
|
||||
});
|
||||
} else {
|
||||
// this should never happen, but we shouldn't fail on it
|
||||
logger.atSevere().log(
|
||||
"%s does not implement DatastoreEntity, which is necessary for SQL replay.",
|
||||
ofyPojo.getClass());
|
||||
try {
|
||||
if (ofyPojo instanceof DatastoreEntity) {
|
||||
DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
|
||||
datastoreEntity
|
||||
.toSqlEntity()
|
||||
.ifPresent(
|
||||
sqlEntity -> {
|
||||
sqlEntity.beforeSqlSaveOnReplay();
|
||||
jpaTm().putIgnoringReadOnly(sqlEntity);
|
||||
});
|
||||
} else {
|
||||
// this should never happen, but we shouldn't fail on it
|
||||
logger.atSevere().log(
|
||||
"%s does not implement DatastoreEntity, which is necessary for SQL replay.",
|
||||
ofyPojo.getClass());
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().log("Error when replaying object %s", ofyPojo);
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,13 +286,18 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
"Skipping SQL delete for kind %s since it is not convertible.", key.getKind());
|
||||
return;
|
||||
}
|
||||
Class<?> entityClass = entityVKey.getKind();
|
||||
// Delete the key iff the class represents a JPA entity that is replicated
|
||||
if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
|
||||
&& !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
|
||||
&& entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
|
||||
ReplaySpecializer.beforeSqlDelete(entityVKey);
|
||||
jpaTm().delete(entityVKey);
|
||||
try {
|
||||
Class<?> entityClass = entityVKey.getKind();
|
||||
// Delete the key iff the class represents a JPA entity that is replicated
|
||||
if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
|
||||
&& !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
|
||||
&& entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
|
||||
ReplaySpecializer.beforeSqlDelete(entityVKey);
|
||||
jpaTm().deleteIgnoringReadOnly(entityVKey);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().log("Error when deleting key %s", entityVKey);
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import com.google.appengine.api.datastore.DatastoreService;
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.EntityTranslator;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
import com.google.common.collect.Streams;
|
||||
@@ -33,7 +33,9 @@ import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Result;
|
||||
import com.googlecode.objectify.util.ResultNow;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
@@ -46,10 +48,10 @@ import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Retrier;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
import javax.inject.Inject;
|
||||
@@ -66,45 +68,57 @@ public class RestoreCommitLogsAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
static final int BLOCK_SIZE = 1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
|
||||
public static final String PATH = "/_dr/task/restoreCommitLogs";
|
||||
static final String DRY_RUN_PARAM = "dryRun";
|
||||
static final String FROM_TIME_PARAM = "fromTime";
|
||||
static final String TO_TIME_PARAM = "toTime";
|
||||
static final String BUCKET_OVERRIDE_PARAM = "gcsBucket";
|
||||
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject @Parameter(DRY_RUN_PARAM) boolean dryRun;
|
||||
@Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime;
|
||||
@Inject @Parameter(TO_TIME_PARAM) DateTime toTime;
|
||||
|
||||
@Inject
|
||||
@Parameter(BUCKET_OVERRIDE_PARAM)
|
||||
Optional<String> gcsBucketOverride;
|
||||
|
||||
@Inject DatastoreService datastoreService;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
|
||||
@Inject
|
||||
@Config("commitLogGcsBucket")
|
||||
String defaultGcsBucket;
|
||||
|
||||
@Inject Retrier retrier;
|
||||
@Inject RestoreCommitLogsAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
checkArgument(
|
||||
RegistryEnvironment.get() == RegistryEnvironment.ALPHA
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.CRASH
|
||||
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
|
||||
"DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
|
||||
!FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
|
||||
"DO NOT RUN IN PRODUCTION OR SANDBOX.");
|
||||
if (dryRun) {
|
||||
logger.atInfo().log("Running in dryRun mode");
|
||||
}
|
||||
List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
|
||||
String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
|
||||
logger.atInfo().log("Restoring from %s.", gcsBucket);
|
||||
List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
|
||||
if (diffFiles.isEmpty()) {
|
||||
logger.atInfo().log("Nothing to restore");
|
||||
return;
|
||||
}
|
||||
Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
|
||||
CommitLogCheckpoint lastCheckpoint = null;
|
||||
for (GcsFileMetadata metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
|
||||
try (InputStream input = Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
for (BlobInfo metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getName());
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
PeekingIterator<ImmutableObject> commitLogs =
|
||||
peekingIterator(createDeserializingIterator(input));
|
||||
peekingIterator(createDeserializingIterator(input, true));
|
||||
lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
|
||||
saveOfy(ImmutableList.of(lastCheckpoint)); // Save the checkpoint itself.
|
||||
while (commitLogs.hasNext()) {
|
||||
|
||||
@@ -28,10 +28,10 @@ import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.domain.RegistryLock;
|
||||
import google.registry.model.eppcommon.Trid;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.schema.domain.RegistryLock;
|
||||
import google.registry.util.AppEngineServiceUtils;
|
||||
import google.registry.util.Retrier;
|
||||
import javax.inject.Inject;
|
||||
|
||||
@@ -15,12 +15,14 @@
|
||||
package google.registry.batch;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.config.RegistryEnvironment.PRODUCTION;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
@@ -28,16 +30,24 @@ import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.flows.poll.PollFlowUtils;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.EppResourceUtils;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex;
|
||||
import google.registry.model.poll.PollMessage;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.reporting.HistoryEntryDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.List;
|
||||
import javax.inject.Inject;
|
||||
|
||||
@@ -46,8 +56,8 @@ import javax.inject.Inject;
|
||||
* the associated ForeignKey and EppResourceIndex entities.
|
||||
*
|
||||
* <p>This only deletes contacts and hosts, NOT domains. To delete domains, use {@link
|
||||
* DeleteLoadTestDataAction} and pass it the TLD(s) that the load test domains were created on. Note
|
||||
* that DeleteLoadTestDataAction is safe enough to run in production whereas this mapreduce is not,
|
||||
* DeleteProberDataAction} and pass it the TLD(s) that the load test domains were created on. Note
|
||||
* that DeleteProberDataAction is safe enough to run in production whereas this mapreduce is not,
|
||||
* but this one does not need to be runnable in production because load testing isn't run against
|
||||
* production.
|
||||
*/
|
||||
@@ -68,15 +78,22 @@ public class DeleteLoadTestDataAction implements Runnable {
|
||||
*/
|
||||
private static final ImmutableSet<String> LOAD_TEST_REGISTRARS = ImmutableSet.of("proxy");
|
||||
|
||||
@Inject
|
||||
@Parameter(PARAM_DRY_RUN)
|
||||
boolean isDryRun;
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
private final boolean isDryRun;
|
||||
private final MapreduceRunner mrRunner;
|
||||
private final Response response;
|
||||
private final Clock clock;
|
||||
|
||||
@Inject
|
||||
DeleteLoadTestDataAction() {}
|
||||
DeleteLoadTestDataAction(
|
||||
@Parameter(PARAM_DRY_RUN) boolean isDryRun,
|
||||
MapreduceRunner mrRunner,
|
||||
Response response,
|
||||
Clock clock) {
|
||||
this.isDryRun = isDryRun;
|
||||
this.mrRunner = mrRunner;
|
||||
this.response = response;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
@@ -87,14 +104,85 @@ public class DeleteLoadTestDataAction implements Runnable {
|
||||
!RegistryEnvironment.get().equals(PRODUCTION),
|
||||
"This mapreduce is not safe to run on PRODUCTION.");
|
||||
|
||||
mrRunner
|
||||
.setJobName("Delete load test data")
|
||||
.setModuleName("backend")
|
||||
.runMapOnly(
|
||||
new DeleteLoadTestDataMapper(isDryRun),
|
||||
ImmutableList.of(
|
||||
createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Delete load test data")
|
||||
.setModuleName("backend")
|
||||
.runMapOnly(
|
||||
new DeleteLoadTestDataMapper(isDryRun),
|
||||
ImmutableList.of(
|
||||
createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
tm().transact(
|
||||
() -> {
|
||||
LOAD_TEST_REGISTRARS.forEach(this::deletePollMessages);
|
||||
tm().loadAllOfStream(ContactResource.class).forEach(this::deleteContact);
|
||||
tm().loadAllOfStream(HostResource.class).forEach(this::deleteHost);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void deletePollMessages(String registrarId) {
|
||||
ImmutableList<PollMessage> pollMessages =
|
||||
PollFlowUtils.createPollMessageQuery(registrarId, END_OF_TIME).list();
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log(
|
||||
"Would delete %d poll messages for registrar %s.", pollMessages.size(), registrarId);
|
||||
} else {
|
||||
pollMessages.forEach(tm()::delete);
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteContact(ContactResource contact) {
|
||||
if (!LOAD_TEST_REGISTRARS.contains(contact.getPersistedCurrentSponsorClientId())) {
|
||||
return;
|
||||
}
|
||||
// We cannot remove contacts from domains in the general case, so we cannot delete contacts
|
||||
// that are linked to domains (since it would break the foreign keys)
|
||||
if (EppResourceUtils.isLinked(contact.createVKey(), clock.nowUtc())) {
|
||||
logger.atWarning().log(
|
||||
"Cannot delete contact with repo ID %s since it is referenced from a domain",
|
||||
contact.getRepoId());
|
||||
return;
|
||||
}
|
||||
deleteResource(contact);
|
||||
}
|
||||
|
||||
private void deleteHost(HostResource host) {
|
||||
if (!LOAD_TEST_REGISTRARS.contains(host.getPersistedCurrentSponsorClientId())) {
|
||||
return;
|
||||
}
|
||||
VKey<HostResource> hostVKey = host.createVKey();
|
||||
// We can remove hosts from linked domains, so we should do so then delete the hosts
|
||||
ImmutableSet<VKey<DomainBase>> linkedDomains =
|
||||
EppResourceUtils.getLinkedDomainKeys(hostVKey, clock.nowUtc(), null);
|
||||
tm().loadByKeys(linkedDomains)
|
||||
.values()
|
||||
.forEach(
|
||||
domain -> {
|
||||
ImmutableSet<VKey<HostResource>> remainingHosts =
|
||||
domain.getNsHosts().stream()
|
||||
.filter(vkey -> !vkey.equals(hostVKey))
|
||||
.collect(toImmutableSet());
|
||||
tm().put(domain.asBuilder().setNameservers(remainingHosts).build());
|
||||
});
|
||||
deleteResource(host);
|
||||
}
|
||||
|
||||
private void deleteResource(EppResource eppResource) {
|
||||
// In SQL, the only objects parented on the resource are poll messages (deleted above) and
|
||||
// history objects.
|
||||
ImmutableList<HistoryEntry> historyObjects =
|
||||
HistoryEntryDao.loadHistoryObjectsForResource(eppResource.createVKey());
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log(
|
||||
"Would delete repo ID %s along with %d history objects",
|
||||
eppResource.getRepoId(), historyObjects.size());
|
||||
} else {
|
||||
historyObjects.forEach(tm()::delete);
|
||||
tm().delete(eppResource);
|
||||
}
|
||||
}
|
||||
|
||||
/** Provides the map method that runs for each existing contact and host entity. */
|
||||
|
||||
@@ -21,8 +21,9 @@ import static google.registry.config.RegistryEnvironment.PRODUCTION;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.model.registry.Registries.getTldsOfType;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
|
||||
import static google.registry.model.tld.Registries.getTldsOfType;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.request.RequestParameters.PARAM_TLDS;
|
||||
@@ -42,27 +43,31 @@ import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.dns.DnsQueue;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.model.CreateAutoTimestamp;
|
||||
import google.registry.model.EppResourceUtils;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.model.tld.Registry.TldType;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import javax.inject.Inject;
|
||||
import org.hibernate.CacheMode;
|
||||
import org.hibernate.ScrollMode;
|
||||
import org.hibernate.ScrollableResults;
|
||||
import org.hibernate.query.Query;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/**
|
||||
* Deletes all prober DomainBases and their subordinate history entries, poll messages, and
|
||||
* billing events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
|
||||
*
|
||||
* <p>See: https://www.youtube.com/watch?v=xuuv0syoHnM
|
||||
* Deletes all prober DomainBases and their subordinate history entries, poll messages, and billing
|
||||
* events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
@@ -73,10 +78,51 @@ public class DeleteProberDataAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
/**
|
||||
* The maximum amount of time we allow a prober domain to be in use.
|
||||
*
|
||||
* <p>In practice, the prober's connection will time out well before this duration. This includes
|
||||
* a decent buffer.
|
||||
*/
|
||||
private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
|
||||
|
||||
/**
|
||||
* The minimum amount of time we want a domain to be "soft deleted".
|
||||
*
|
||||
* <p>The domain has to remain soft deleted for at least enough time for the DNS task to run and
|
||||
* remove it from DNS itself. This is probably on the order of minutes.
|
||||
*/
|
||||
private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
|
||||
|
||||
private static final DnsQueue dnsQueue = DnsQueue.create();
|
||||
|
||||
// Domains to delete must:
|
||||
// 1. Be in one of the prober TLDs
|
||||
// 2. Not be a nic domain
|
||||
// 3. Have no subordinate hosts
|
||||
// 4. Not still be used (within an hour of creation time)
|
||||
// 5. Either be active (creationTime <= now < deletionTime) or have been deleted a while ago (this
|
||||
// prevents accidental double-map with the same key from immediately deleting active domains)
|
||||
//
|
||||
// Note: creationTime must be compared to a Java object (CreateAutoTimestamp) but deletionTime can
|
||||
// be compared directly to the SQL timestamp (it's a DateTime)
|
||||
private static final String DOMAIN_QUERY_STRING =
|
||||
"FROM Domain d WHERE d.tld IN :tlds AND d.fullyQualifiedDomainName NOT LIKE 'nic.%' AND"
|
||||
+ " (d.subordinateHosts IS EMPTY OR d.subordinateHosts IS NULL) AND d.creationTime <"
|
||||
+ " :creationTimeCutoff AND ((d.creationTime <= :nowAutoTimestamp AND d.deletionTime >"
|
||||
+ " current_timestamp()) OR d.deletionTime < :nowMinusSoftDeleteDelay) ORDER BY d.repoId";
|
||||
|
||||
/** Number of domains to retrieve and delete per SQL transaction. */
|
||||
private static final int BATCH_SIZE = 1000;
|
||||
|
||||
@Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
|
||||
/** List of TLDs to work on. If empty - will work on all TLDs that end with .test. */
|
||||
@Inject @Parameter(PARAM_TLDS) ImmutableSet<String> tlds;
|
||||
@Inject @Config("registryAdminClientId") String registryAdminClientId;
|
||||
|
||||
@Inject
|
||||
@Config("registryAdminClientId")
|
||||
String registryAdminRegistrarId;
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject DeleteProberDataAction() {}
|
||||
@@ -84,25 +130,14 @@ public class DeleteProberDataAction implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
checkState(
|
||||
!Strings.isNullOrEmpty(registryAdminClientId),
|
||||
!Strings.isNullOrEmpty(registryAdminRegistrarId),
|
||||
"Registry admin client ID must be configured for prober data deletion to work");
|
||||
mrRunner
|
||||
.setJobName("Delete prober data")
|
||||
.setModuleName("backend")
|
||||
.runMapOnly(
|
||||
new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun, registryAdminClientId),
|
||||
ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
}
|
||||
|
||||
private ImmutableSet<String> getProberRoidSuffixes() {
|
||||
checkArgument(
|
||||
!PRODUCTION.equals(RegistryEnvironment.get())
|
||||
|| tlds.stream().allMatch(tld -> tld.endsWith(".test")),
|
||||
"On production, can only work on TLDs that end with .test");
|
||||
ImmutableSet<String> deletableTlds =
|
||||
getTldsOfType(TldType.TEST)
|
||||
.stream()
|
||||
getTldsOfType(TldType.TEST).stream()
|
||||
.filter(tld -> tlds.isEmpty() ? tld.endsWith(".test") : tlds.contains(tld))
|
||||
.collect(toImmutableSet());
|
||||
checkArgument(
|
||||
@@ -110,10 +145,161 @@ public class DeleteProberDataAction implements Runnable {
|
||||
"If tlds are given, they must all exist and be TEST tlds. Given: %s, not found: %s",
|
||||
tlds,
|
||||
Sets.difference(tlds, deletableTlds));
|
||||
return deletableTlds
|
||||
.stream()
|
||||
.map(tld -> Registry.get(tld).getRoidSuffix())
|
||||
.collect(toImmutableSet());
|
||||
ImmutableSet<String> proberRoidSuffixes =
|
||||
deletableTlds.stream()
|
||||
.map(tld -> Registry.get(tld).getRoidSuffix())
|
||||
.collect(toImmutableSet());
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Delete prober data")
|
||||
.setModuleName("backend")
|
||||
.runMapOnly(
|
||||
new DeleteProberDataMapper(proberRoidSuffixes, isDryRun, registryAdminRegistrarId),
|
||||
ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
runSqlJob(deletableTlds);
|
||||
}
|
||||
}
|
||||
|
||||
private void runSqlJob(ImmutableSet<String> deletableTlds) {
|
||||
AtomicInteger softDeletedDomains = new AtomicInteger();
|
||||
AtomicInteger hardDeletedDomains = new AtomicInteger();
|
||||
jpaTm().transact(() -> processDomains(deletableTlds, softDeletedDomains, hardDeletedDomains));
|
||||
logger.atInfo().log(
|
||||
"%s %d domains.",
|
||||
isDryRun ? "Would have soft-deleted" : "Soft-deleted", softDeletedDomains.get());
|
||||
logger.atInfo().log(
|
||||
"%s %d domains.",
|
||||
isDryRun ? "Would have hard-deleted" : "Hard-deleted", hardDeletedDomains.get());
|
||||
}
|
||||
|
||||
private void processDomains(
|
||||
ImmutableSet<String> deletableTlds,
|
||||
AtomicInteger softDeletedDomains,
|
||||
AtomicInteger hardDeletedDomains) {
|
||||
DateTime now = tm().getTransactionTime();
|
||||
// Scroll through domains, soft-deleting as necessary (very few will be soft-deleted) and
|
||||
// keeping track of which domains to hard-delete (there can be many, so we batch them up)
|
||||
ScrollableResults scrollableResult =
|
||||
jpaTm()
|
||||
.query(DOMAIN_QUERY_STRING, DomainBase.class)
|
||||
.setParameter("tlds", deletableTlds)
|
||||
.setParameter(
|
||||
"creationTimeCutoff", CreateAutoTimestamp.create(now.minus(DOMAIN_USED_DURATION)))
|
||||
.setParameter("nowMinusSoftDeleteDelay", now.minus(SOFT_DELETE_DELAY))
|
||||
.setParameter("nowAutoTimestamp", CreateAutoTimestamp.create(now))
|
||||
.unwrap(Query.class)
|
||||
.setCacheMode(CacheMode.IGNORE)
|
||||
.scroll(ScrollMode.FORWARD_ONLY);
|
||||
ImmutableList.Builder<String> domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
|
||||
ImmutableList.Builder<String> hostNamesToHardDelete = new ImmutableList.Builder<>();
|
||||
for (int i = 1; scrollableResult.next(); i = (i + 1) % BATCH_SIZE) {
|
||||
DomainBase domain = (DomainBase) scrollableResult.get(0);
|
||||
processDomain(
|
||||
domain,
|
||||
domainRepoIdsToHardDelete,
|
||||
hostNamesToHardDelete,
|
||||
softDeletedDomains,
|
||||
hardDeletedDomains);
|
||||
// Batch the deletion and DB flush + session clearing so we don't OOM
|
||||
if (i == 0) {
|
||||
hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
|
||||
domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
|
||||
hostNamesToHardDelete = new ImmutableList.Builder<>();
|
||||
jpaTm().getEntityManager().flush();
|
||||
jpaTm().getEntityManager().clear();
|
||||
}
|
||||
}
|
||||
// process the remainder
|
||||
hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
|
||||
}
|
||||
|
||||
private void processDomain(
|
||||
DomainBase domain,
|
||||
ImmutableList.Builder<String> domainRepoIdsToHardDelete,
|
||||
ImmutableList.Builder<String> hostNamesToHardDelete,
|
||||
AtomicInteger softDeletedDomains,
|
||||
AtomicInteger hardDeletedDomains) {
|
||||
// If the domain is still active, that means that the prober encountered a failure and did not
|
||||
// successfully soft-delete the domain (thus leaving its DNS entry published). We soft-delete
|
||||
// it now so that the DNS entry can be handled. The domain will then be hard-deleted the next
|
||||
// time the job is run.
|
||||
if (EppResourceUtils.isActive(domain, tm().getTransactionTime())) {
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log(
|
||||
"Would soft-delete the active domain: %s (%s)",
|
||||
domain.getDomainName(), domain.getRepoId());
|
||||
} else {
|
||||
softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue);
|
||||
}
|
||||
softDeletedDomains.incrementAndGet();
|
||||
} else {
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log(
|
||||
"Would hard-delete the non-active domain: %s (%s) and its dependents",
|
||||
domain.getDomainName(), domain.getRepoId());
|
||||
} else {
|
||||
domainRepoIdsToHardDelete.add(domain.getRepoId());
|
||||
hostNamesToHardDelete.addAll(domain.getSubordinateHosts());
|
||||
}
|
||||
hardDeletedDomains.incrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
private void hardDeleteDomainsAndHosts(
|
||||
ImmutableList<String> domainRepoIds, ImmutableList<String> hostNames) {
|
||||
jpaTm()
|
||||
.query("DELETE FROM Host WHERE fullyQualifiedHostName IN :hostNames")
|
||||
.setParameter("hostNames", hostNames)
|
||||
.executeUpdate();
|
||||
jpaTm()
|
||||
.query("DELETE FROM BillingEvent WHERE domainRepoId IN :repoIds")
|
||||
.setParameter("repoIds", domainRepoIds)
|
||||
.executeUpdate();
|
||||
jpaTm()
|
||||
.query("DELETE FROM BillingRecurrence WHERE domainRepoId IN :repoIds")
|
||||
.setParameter("repoIds", domainRepoIds)
|
||||
.executeUpdate();
|
||||
jpaTm()
|
||||
.query("DELETE FROM BillingCancellation WHERE domainRepoId IN :repoIds")
|
||||
.setParameter("repoIds", domainRepoIds)
|
||||
.executeUpdate();
|
||||
jpaTm()
|
||||
.query("DELETE FROM DomainHistory WHERE domainRepoId IN :repoIds")
|
||||
.setParameter("repoIds", domainRepoIds)
|
||||
.executeUpdate();
|
||||
jpaTm()
|
||||
.query("DELETE FROM PollMessage WHERE domainRepoId IN :repoIds")
|
||||
.setParameter("repoIds", domainRepoIds)
|
||||
.executeUpdate();
|
||||
jpaTm()
|
||||
.query("DELETE FROM Domain WHERE repoId IN :repoIds")
|
||||
.setParameter("repoIds", domainRepoIds)
|
||||
.executeUpdate();
|
||||
}
|
||||
|
||||
// Take a DNS queue + admin registrar id as input so that it can be called from the mapper as well
|
||||
private static void softDeleteDomain(
|
||||
DomainBase domain, String registryAdminRegistrarId, DnsQueue localDnsQueue) {
|
||||
DomainBase deletedDomain =
|
||||
domain.asBuilder().setDeletionTime(tm().getTransactionTime()).setStatusValues(null).build();
|
||||
DomainHistory historyEntry =
|
||||
new DomainHistory.Builder()
|
||||
.setDomain(domain)
|
||||
.setType(DOMAIN_DELETE)
|
||||
.setModificationTime(tm().getTransactionTime())
|
||||
.setBySuperuser(true)
|
||||
.setReason("Deletion of prober data")
|
||||
.setClientId(registryAdminRegistrarId)
|
||||
.build();
|
||||
// Note that we don't bother handling grace periods, billing events, pending transfers, poll
|
||||
// messages, or auto-renews because those will all be hard-deleted the next time the job runs
|
||||
// anyway.
|
||||
tm().putAllWithoutBackup(ImmutableList.of(deletedDomain, historyEntry));
|
||||
// updating foreign keys is a no-op in SQL
|
||||
updateForeignKeyIndexDeletionTime(deletedDomain);
|
||||
localDnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
|
||||
}
|
||||
|
||||
/** Provides the map method that runs for each existing DomainBase entity. */
|
||||
@@ -122,32 +308,17 @@ public class DeleteProberDataAction implements Runnable {
|
||||
private static final DnsQueue dnsQueue = DnsQueue.create();
|
||||
private static final long serialVersionUID = -7724537393697576369L;
|
||||
|
||||
/**
|
||||
* The maximum amount of time we allow a prober domain to be in use.
|
||||
*
|
||||
* In practice, the prober's connection will time out well before this duration. This includes a
|
||||
* decent buffer.
|
||||
*
|
||||
*/
|
||||
private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
|
||||
|
||||
/**
|
||||
* The minimum amount of time we want a domain to be "soft deleted".
|
||||
*
|
||||
* The domain has to remain soft deleted for at least enough time for the DNS task to run and
|
||||
* remove it from DNS itself. This is probably on the order of minutes.
|
||||
*/
|
||||
private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
|
||||
|
||||
private final ImmutableSet<String> proberRoidSuffixes;
|
||||
private final Boolean isDryRun;
|
||||
private final String registryAdminClientId;
|
||||
private final String registryAdminRegistrarId;
|
||||
|
||||
public DeleteProberDataMapper(
|
||||
ImmutableSet<String> proberRoidSuffixes, Boolean isDryRun, String registryAdminClientId) {
|
||||
ImmutableSet<String> proberRoidSuffixes,
|
||||
Boolean isDryRun,
|
||||
String registryAdminRegistrarId) {
|
||||
this.proberRoidSuffixes = proberRoidSuffixes;
|
||||
this.isDryRun = isDryRun;
|
||||
this.registryAdminClientId = registryAdminClientId;
|
||||
this.registryAdminRegistrarId = registryAdminRegistrarId;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -203,7 +374,7 @@ public class DeleteProberDataAction implements Runnable {
|
||||
logger.atInfo().log(
|
||||
"Would soft-delete the active domain: %s (%s)", domainName, domainKey);
|
||||
} else {
|
||||
softDeleteDomain(domain);
|
||||
tm().transact(() -> softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue));
|
||||
}
|
||||
getContext().incrementCounter("domains soft-deleted");
|
||||
return;
|
||||
@@ -223,8 +394,7 @@ public class DeleteProberDataAction implements Runnable {
|
||||
tm().transact(
|
||||
() -> {
|
||||
// This ancestor query selects all descendant HistoryEntries, BillingEvents,
|
||||
// PollMessages,
|
||||
// and TLD-specific entities, as well as the domain itself.
|
||||
// PollMessages, and TLD-specific entities, as well as the domain itself.
|
||||
List<Key<Object>> domainAndDependentKeys =
|
||||
auditedOfy().load().ancestor(domainKey).keys().list();
|
||||
ImmutableSet<Key<?>> allKeys =
|
||||
@@ -243,32 +413,5 @@ public class DeleteProberDataAction implements Runnable {
|
||||
getContext().incrementCounter("domains hard-deleted");
|
||||
getContext().incrementCounter("total entities hard-deleted", entitiesDeleted);
|
||||
}
|
||||
|
||||
private void softDeleteDomain(final DomainBase domain) {
|
||||
tm().transactNew(
|
||||
() -> {
|
||||
DomainBase deletedDomain =
|
||||
domain
|
||||
.asBuilder()
|
||||
.setDeletionTime(tm().getTransactionTime())
|
||||
.setStatusValues(null)
|
||||
.build();
|
||||
DomainHistory historyEntry =
|
||||
new DomainHistory.Builder()
|
||||
.setDomain(domain)
|
||||
.setType(DOMAIN_DELETE)
|
||||
.setModificationTime(tm().getTransactionTime())
|
||||
.setBySuperuser(true)
|
||||
.setReason("Deletion of prober data")
|
||||
.setClientId(registryAdminClientId)
|
||||
.build();
|
||||
// Note that we don't bother handling grace periods, billing events, pending
|
||||
// transfers, poll messages, or auto-renews because these will all be hard-deleted
|
||||
// the next time the mapreduce runs anyway.
|
||||
tm().putAll(deletedDomain, historyEntry);
|
||||
updateForeignKeyIndexDeletionTime(deletedDomain);
|
||||
dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,9 +52,9 @@ import google.registry.model.common.Cursor;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.Period;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.reporting.DomainTransactionRecord;
|
||||
import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
@@ -127,10 +127,11 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
jpaTm()
|
||||
.query(
|
||||
"FROM BillingRecurrence "
|
||||
+ "WHERE event_time <= :executeTime "
|
||||
+ "AND event_time < recurrence_end_time",
|
||||
+ "WHERE eventTime <= :executeTime "
|
||||
+ "AND eventTime < recurrenceEndTime "
|
||||
+ "ORDER BY id ASC",
|
||||
Recurring.class)
|
||||
.setParameter("executeTime", executeTime.toDate())
|
||||
.setParameter("executeTime", executeTime)
|
||||
// Need to get a list from the transaction and then convert it to a stream
|
||||
// for further processing. If we get a stream directly, each elements gets
|
||||
// processed downstream eagerly but Hibernate returns a
|
||||
|
||||
@@ -23,6 +23,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
|
||||
import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.getLinkedDomainKeys;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
import static google.registry.model.EppResourceUtils.isDeleted;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
@@ -69,6 +70,7 @@ import java.util.logging.Level;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
@@ -86,6 +88,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
||||
@Inject Clock clock;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject @Named(QUEUE_ASYNC_HOST_RENAME) Queue pullQueue;
|
||||
|
||||
@Inject DnsQueue dnsQueue;
|
||||
@Inject RequestStatusChecker requestStatusChecker;
|
||||
@Inject Response response;
|
||||
@Inject Retrier retrier;
|
||||
@@ -153,7 +157,39 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
||||
} else {
|
||||
logger.atInfo().log(
|
||||
"Processing asynchronous DNS refresh for renamed hosts: %s", hostKeys.build());
|
||||
runMapreduce(refreshRequests, lock);
|
||||
if (tm().isOfy()) {
|
||||
runMapreduce(refreshRequests, lock);
|
||||
} else {
|
||||
try {
|
||||
refreshRequests.stream()
|
||||
.flatMap(
|
||||
request ->
|
||||
getLinkedDomainKeys(request.hostKey(), request.lastUpdateTime(), null)
|
||||
.stream())
|
||||
.distinct()
|
||||
.map(domainKey -> tm().transact(() -> tm().loadByKey(domainKey).getDomainName()))
|
||||
.forEach(
|
||||
domainName -> {
|
||||
retrier.callWithRetry(
|
||||
() -> dnsQueue.addDomainRefreshTask(domainName),
|
||||
TransientFailureException.class);
|
||||
logger.atInfo().log("Enqueued DNS refresh for domain %s.", domainName);
|
||||
});
|
||||
deleteTasksWithRetry(
|
||||
refreshRequests,
|
||||
getQueue(QUEUE_ASYNC_HOST_RENAME),
|
||||
asyncTaskMetrics,
|
||||
retrier,
|
||||
OperationResult.SUCCESS);
|
||||
} catch (Throwable t) {
|
||||
String message = "Error refreshing DNS on host rename.";
|
||||
logger.atSevere().withCause(t).log(message);
|
||||
response.setPayload(message);
|
||||
response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR);
|
||||
} finally {
|
||||
lock.get().release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,16 +28,16 @@ import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.RegistryLock;
|
||||
import google.registry.model.eppcommon.StatusValue;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarContact;
|
||||
import google.registry.model.registry.RegistryLockDao;
|
||||
import google.registry.model.tld.RegistryLockDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.schema.domain.RegistryLock;
|
||||
import google.registry.tools.DomainLockUtils;
|
||||
import google.registry.util.DateTimeUtils;
|
||||
import google.registry.util.EmailMessage;
|
||||
|
||||
@@ -54,6 +54,8 @@ import javax.inject.Inject;
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/resaveAllEppResources",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
// No longer needed in SQL. Subject to future removal.
|
||||
@Deprecated
|
||||
public class ResaveAllEppResourcesAction implements Runnable {
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.batch;
|
||||
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
|
||||
import static org.apache.http.HttpStatus.SC_OK;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.flows.certs.CertificateChecker;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarContact;
|
||||
import google.registry.model.registrar.RegistrarContact.Type;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.EmailMessage;
|
||||
import google.registry.util.SendEmailService;
|
||||
import java.util.Date;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
import javax.mail.internet.AddressException;
|
||||
import javax.mail.internet.InternetAddress;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
|
||||
/** An action that sends notification emails to registrars whose certificates are expiring soon. */
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = SendExpiringCertificateNotificationEmailAction.PATH,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
public class SendExpiringCertificateNotificationEmailAction implements Runnable {
|
||||
public static final String PATH = "/_dr/task/sendExpiringCertificateNotificationEmail";
|
||||
/**
|
||||
* Used as an offset when storing the last notification email sent date.
|
||||
*
|
||||
* <p>This is used to handle edges cases when the update happens in between the day switch. For
|
||||
* instance,if the job starts at 2:00 am every day and it finishes at 2:03 of the same day, then
|
||||
* next day at 2am, the date difference will be less than a day, which will lead to the date
|
||||
* difference between two successive email sent date being the expected email interval days + 1;
|
||||
*/
|
||||
protected static final Duration UPDATE_TIME_OFFSET = Duration.standardMinutes(10);
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd");
|
||||
|
||||
private final CertificateChecker certificateChecker;
|
||||
private final String expirationWarningEmailBodyText;
|
||||
private final SendEmailService sendEmailService;
|
||||
private final String expirationWarningEmailSubjectText;
|
||||
private final InternetAddress gSuiteOutgoingEmailAddress;
|
||||
private final Response response;
|
||||
|
||||
@Inject
|
||||
public SendExpiringCertificateNotificationEmailAction(
|
||||
@Config("expirationWarningEmailBodyText") String expirationWarningEmailBodyText,
|
||||
@Config("expirationWarningEmailSubjectText") String expirationWarningEmailSubjectText,
|
||||
@Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress,
|
||||
SendEmailService sendEmailService,
|
||||
CertificateChecker certificateChecker,
|
||||
Response response) {
|
||||
this.certificateChecker = certificateChecker;
|
||||
this.expirationWarningEmailSubjectText = expirationWarningEmailSubjectText;
|
||||
this.sendEmailService = sendEmailService;
|
||||
this.gSuiteOutgoingEmailAddress = gSuiteOutgoingEmailAddress;
|
||||
this.expirationWarningEmailBodyText = expirationWarningEmailBodyText;
|
||||
this.response = response;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
|
||||
try {
|
||||
sendNotificationEmails();
|
||||
response.setStatus(SC_OK);
|
||||
} catch (Exception e) {
|
||||
logger.atWarning().withCause(e).log(
|
||||
"Exception thrown when sending expiring certificate notification emails.");
|
||||
response.setStatus(SC_INTERNAL_SERVER_ERROR);
|
||||
response.setPayload(String.format("Exception thrown with cause: %s", e));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list of registrars that should receive expiring notification emails. There are two
|
||||
* certificates that should be considered (the main certificate and failOver certificate). The
|
||||
* registrars should receive notifications if one of the certificate checks returns true.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
|
||||
logger.atInfo().log(
|
||||
"Getting a list of registrars that should receive expiring notification emails.");
|
||||
return Streams.stream(Registrar.loadAllCached())
|
||||
.map(
|
||||
registrar ->
|
||||
RegistrarInfo.create(
|
||||
registrar,
|
||||
registrar.getClientCertificate().isPresent()
|
||||
&& certificateChecker.shouldReceiveExpiringNotification(
|
||||
registrar.getLastExpiringCertNotificationSentDate(),
|
||||
registrar.getClientCertificate().get()),
|
||||
registrar.getFailoverClientCertificate().isPresent()
|
||||
&& certificateChecker.shouldReceiveExpiringNotification(
|
||||
registrar.getLastExpiringFailoverCertNotificationSentDate(),
|
||||
registrar.getFailoverClientCertificate().get())))
|
||||
.filter(
|
||||
registrarInfo ->
|
||||
registrarInfo.isCertExpiring() || registrarInfo.isFailOverCertExpiring())
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a notification email to the registrar regarding the expiring certificate and returns true
|
||||
* if it's sent successfully.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
boolean sendNotificationEmail(
|
||||
Registrar registrar,
|
||||
DateTime lastExpiringCertNotificationSentDate,
|
||||
CertificateType certificateType,
|
||||
Optional<String> certificate) {
|
||||
if (!certificate.isPresent()
|
||||
|| !certificateChecker.shouldReceiveExpiringNotification(
|
||||
lastExpiringCertNotificationSentDate, certificate.get())) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
|
||||
Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
|
||||
logger.atInfo().log(
|
||||
"Registrar %s should receive an email that its %s SSL certificate will expire on %s.",
|
||||
registrar.getRegistrarName(),
|
||||
certificateType.getDisplayName(),
|
||||
expirationDate.toString());
|
||||
if (recipients.isEmpty()) {
|
||||
logger.atWarning().log(
|
||||
"Registrar %s contains no email addresses to receive notification email.",
|
||||
registrar.getRegistrarName());
|
||||
return false;
|
||||
}
|
||||
sendEmailService.sendEmail(
|
||||
EmailMessage.newBuilder()
|
||||
.setFrom(gSuiteOutgoingEmailAddress)
|
||||
.setSubject(expirationWarningEmailSubjectText)
|
||||
.setBody(
|
||||
getEmailBody(
|
||||
registrar.getRegistrarName(),
|
||||
certificateType,
|
||||
expirationDate,
|
||||
registrar.getClientId()))
|
||||
.setRecipients(recipients)
|
||||
.setCcs(getEmailAddresses(registrar, Type.ADMIN))
|
||||
.build());
|
||||
/*
|
||||
* A duration time offset is used here to ensure that date comparison between two
|
||||
* successive dates is always greater than 1 day. This date is set as last updated date,
|
||||
* for applicable certificate.
|
||||
*/
|
||||
updateLastNotificationSentDate(
|
||||
registrar,
|
||||
DateTime.now(UTC).minusMinutes((int) UPDATE_TIME_OFFSET.getStandardMinutes()),
|
||||
certificateType);
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"Failed to send expiring certificate notification email to registrar %s.",
|
||||
registrar.getRegistrarName()));
|
||||
}
|
||||
}
|
||||
|
||||
/** Updates the last notification sent date in database. */
|
||||
@VisibleForTesting
|
||||
void updateLastNotificationSentDate(
|
||||
Registrar registrar, DateTime now, CertificateType certificateType) {
|
||||
try {
|
||||
tm().transact(
|
||||
() -> {
|
||||
Registrar.Builder newRegistrar = tm().loadByEntity(registrar).asBuilder();
|
||||
switch (certificateType) {
|
||||
case PRIMARY:
|
||||
newRegistrar.setLastExpiringCertNotificationSentDate(now);
|
||||
tm().put(newRegistrar.build());
|
||||
logger.atInfo().log(
|
||||
"Updated last notification email sent date to %s for %s certificate of "
|
||||
+ "registrar %s.",
|
||||
DATE_FORMATTER.print(now),
|
||||
certificateType.getDisplayName(),
|
||||
registrar.getRegistrarName());
|
||||
break;
|
||||
case FAILOVER:
|
||||
newRegistrar.setLastExpiringFailoverCertNotificationSentDate(now);
|
||||
tm().put(newRegistrar.build());
|
||||
logger.atInfo().log(
|
||||
"Updated last notification email sent date to %s for %s certificate of "
|
||||
+ "registrar %s.",
|
||||
DATE_FORMATTER.print(now),
|
||||
certificateType.getDisplayName(),
|
||||
registrar.getRegistrarName());
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException(
|
||||
String.format(
|
||||
"Unsupported certificate type: %s being passed in when updating "
|
||||
+ "the last notification sent date to registrar %s.",
|
||||
certificateType.toString(), registrar.getRegistrarName()));
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"Failed to update the last notification sent date to Registrar %s for the %s "
|
||||
+ "certificate.",
|
||||
registrar.getRegistrarName(), certificateType.getDisplayName()));
|
||||
}
|
||||
}
|
||||
|
||||
/** Sends notification emails to registrars with expiring certificates. */
|
||||
@VisibleForTesting
|
||||
int sendNotificationEmails() {
|
||||
int emailsSent = 0;
|
||||
for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
|
||||
Registrar registrar = registrarInfo.registrar();
|
||||
if (registrarInfo.isCertExpiring()) {
|
||||
sendNotificationEmail(
|
||||
registrar,
|
||||
registrar.getLastExpiringCertNotificationSentDate(),
|
||||
CertificateType.PRIMARY,
|
||||
registrar.getClientCertificate());
|
||||
emailsSent++;
|
||||
}
|
||||
if (registrarInfo.isFailOverCertExpiring()) {
|
||||
sendNotificationEmail(
|
||||
registrar,
|
||||
registrar.getLastExpiringFailoverCertNotificationSentDate(),
|
||||
CertificateType.FAILOVER,
|
||||
registrar.getFailoverClientCertificate());
|
||||
emailsSent++;
|
||||
}
|
||||
}
|
||||
logger.atInfo().log(
|
||||
"Attempted to send %d expiring certificate notification emails.", emailsSent);
|
||||
return emailsSent;
|
||||
}
|
||||
|
||||
/** Returns a list of email addresses of the registrar that should receive a notification email */
|
||||
@VisibleForTesting
|
||||
ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
|
||||
ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);
|
||||
ImmutableSet.Builder<InternetAddress> recipientEmails = new ImmutableSet.Builder<>();
|
||||
for (RegistrarContact contact : contacts) {
|
||||
try {
|
||||
recipientEmails.add(new InternetAddress(contact.getEmailAddress()));
|
||||
} catch (AddressException e) {
|
||||
logger.atWarning().withCause(e).log(
|
||||
"Registrar Contact email address %s of Registrar %s is invalid; skipping.",
|
||||
contact.getEmailAddress(), registrar.getRegistrarName());
|
||||
}
|
||||
}
|
||||
return recipientEmails.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates email content by taking registrar name, certificate type and expiration date as
|
||||
* parameters.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
@SuppressWarnings("lgtm[java/dereferenced-value-may-be-null]")
|
||||
String getEmailBody(
|
||||
String registrarName, CertificateType type, Date expirationDate, String registrarId) {
|
||||
checkArgumentNotNull(expirationDate, "Expiration date cannot be null");
|
||||
checkArgumentNotNull(type, "Certificate type cannot be null");
|
||||
checkArgumentNotNull(registrarId, "Registrar Id cannot be null");
|
||||
return String.format(
|
||||
expirationWarningEmailBodyText,
|
||||
registrarName,
|
||||
type.getDisplayName(),
|
||||
DATE_FORMATTER.print(new DateTime(expirationDate)),
|
||||
registrarId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Certificate types for X509Certificate.
|
||||
*
|
||||
* <p><b>Note:</b> These types are only used to indicate the type of expiring certificate in
|
||||
* notification emails.
|
||||
*/
|
||||
protected enum CertificateType {
|
||||
PRIMARY("primary"),
|
||||
FAILOVER("fail-over");
|
||||
|
||||
private final String displayName;
|
||||
|
||||
CertificateType(String displayName) {
|
||||
this.displayName = displayName;
|
||||
}
|
||||
|
||||
public String getDisplayName() {
|
||||
return displayName;
|
||||
}
|
||||
}
|
||||
|
||||
@AutoValue
|
||||
public abstract static class RegistrarInfo {
|
||||
static RegistrarInfo create(
|
||||
Registrar registrar, boolean isCertExpiring, boolean isFailOverCertExpiring) {
|
||||
return new AutoValue_SendExpiringCertificateNotificationEmailAction_RegistrarInfo(
|
||||
registrar, isCertExpiring, isFailOverCertExpiring);
|
||||
}
|
||||
|
||||
public abstract Registrar registrar();
|
||||
|
||||
public abstract boolean isCertExpiring();
|
||||
|
||||
public abstract boolean isFailOverCertExpiring();
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.persistence.PersistenceModule.SchemaManagerConnection;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
@@ -48,22 +48,18 @@ import javax.inject.Inject;
|
||||
public class WipeOutCloudSqlAction implements Runnable {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
// As a short-lived class, hardcode allowed projects here instead of using config files.
|
||||
private static final ImmutableSet<String> ALLOWED_PROJECTS =
|
||||
ImmutableSet.of("domain-registry-qa");
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
private final String projectId;
|
||||
private final Supplier<Connection> connectionSupplier;
|
||||
private final Response response;
|
||||
private final Retrier retrier;
|
||||
|
||||
@Inject
|
||||
WipeOutCloudSqlAction(
|
||||
@Config("projectId") String projectId,
|
||||
@SchemaManagerConnection Supplier<Connection> connectionSupplier,
|
||||
Response response,
|
||||
Retrier retrier) {
|
||||
this.projectId = projectId;
|
||||
this.connectionSupplier = connectionSupplier;
|
||||
this.response = response;
|
||||
this.retrier = retrier;
|
||||
@@ -73,9 +69,9 @@ public class WipeOutCloudSqlAction implements Runnable {
|
||||
public void run() {
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
|
||||
if (!ALLOWED_PROJECTS.contains(projectId)) {
|
||||
if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
|
||||
response.setStatus(SC_FORBIDDEN);
|
||||
response.setPayload("Wipeout is not allowed in " + projectId);
|
||||
response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -90,11 +86,11 @@ public class WipeOutCloudSqlAction implements Runnable {
|
||||
},
|
||||
e -> !(e instanceof SQLException));
|
||||
response.setStatus(SC_OK);
|
||||
response.setPayload("Wiped out Cloud SQL in " + projectId);
|
||||
response.setPayload("Wiped out Cloud SQL in " + RegistryEnvironment.get());
|
||||
} catch (RuntimeException e) {
|
||||
logger.atSevere().withCause(e).log("Failed to wipe out Cloud SQL data.");
|
||||
response.setStatus(SC_INTERNAL_SERVER_ERROR);
|
||||
response.setPayload("Failed to wipe out Cloud SQL in " + projectId);
|
||||
response.setPayload("Failed to wipe out Cloud SQL in " + RegistryEnvironment.get());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
@@ -49,9 +50,8 @@ public class WipeoutDatastoreAction implements Runnable {
|
||||
|
||||
private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";
|
||||
|
||||
// As a short-lived class, hardcode allowed projects here instead of using config files.
|
||||
private static final ImmutableSet<String> ALLOWED_PROJECTS =
|
||||
ImmutableSet.of("domain-registry-qa");
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
private final String projectId;
|
||||
private final String jobRegion;
|
||||
@@ -80,9 +80,9 @@ public class WipeoutDatastoreAction implements Runnable {
|
||||
public void run() {
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
|
||||
if (!ALLOWED_PROJECTS.contains(projectId)) {
|
||||
if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
|
||||
response.setStatus(SC_FORBIDDEN);
|
||||
response.setPayload("Wipeout is not allowed in " + projectId);
|
||||
response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -18,17 +18,23 @@ import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
|
||||
import google.registry.backup.AppEngineEnvironment;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.persistence.transaction.CriteriaQueryBuilder;
|
||||
import google.registry.persistence.transaction.JpaTransactionManager;
|
||||
import java.io.Serializable;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.metrics.Counter;
|
||||
import org.apache.beam.sdk.metrics.Metrics;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.transforms.Create;
|
||||
import org.apache.beam.sdk.transforms.DoFn;
|
||||
import org.apache.beam.sdk.transforms.ParDo;
|
||||
|
||||
/** Toy pipeline that demonstrates how to use {@link JpaTransactionManager} in BEAM pipelines. */
|
||||
/**
|
||||
* Toy pipeline that demonstrates how to use {@link JpaTransactionManager} in BEAM pipelines.
|
||||
*
|
||||
* <p>This pipeline may also be used as an integration test for {@link RegistryJpaIO.Read} in a
|
||||
* project with realistic data.
|
||||
*/
|
||||
public class JpaDemoPipeline implements Serializable {
|
||||
|
||||
public static void main(String[] args) {
|
||||
@@ -38,23 +44,16 @@ public class JpaDemoPipeline implements Serializable {
|
||||
|
||||
Pipeline pipeline = Pipeline.create(options);
|
||||
pipeline
|
||||
.apply("Start", Create.of((Void) null))
|
||||
.apply(
|
||||
"Generate Elements",
|
||||
ParDo.of(
|
||||
new DoFn<Void, Void>() {
|
||||
@ProcessElement
|
||||
public void processElement(OutputReceiver<Void> output) {
|
||||
for (int i = 0; i < 500; i++) {
|
||||
output.output(null);
|
||||
}
|
||||
}
|
||||
}))
|
||||
"Read contacts",
|
||||
RegistryJpaIO.read(
|
||||
() -> CriteriaQueryBuilder.create(ContactResource.class).build(),
|
||||
ContactResource::getRepoId))
|
||||
.apply(
|
||||
"Make Query",
|
||||
"Count Contacts",
|
||||
ParDo.of(
|
||||
new DoFn<Void, Void>() {
|
||||
private Counter counter = Metrics.counter("Demo", "Read");
|
||||
new DoFn<String, Void>() {
|
||||
private Counter counter = Metrics.counter("Contacts", "Read");
|
||||
|
||||
@ProcessElement
|
||||
public void processElement() {
|
||||
|
||||
@@ -21,14 +21,16 @@ import com.google.auto.value.AutoValue;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Streams;
|
||||
import google.registry.backup.AppEngineEnvironment;
|
||||
import google.registry.beam.common.RegistryQuery.QueryComposerFactory;
|
||||
import google.registry.beam.common.RegistryQuery.RegistryQueryFactory;
|
||||
import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
|
||||
import google.registry.model.ofy.ObjectifyService;
|
||||
import google.registry.model.replay.SqlEntity;
|
||||
import google.registry.persistence.transaction.JpaTransactionManager;
|
||||
import google.registry.persistence.transaction.TransactionManagerFactory;
|
||||
import java.io.Serializable;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.persistence.criteria.CriteriaQuery;
|
||||
import org.apache.beam.sdk.coders.Coder;
|
||||
import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
@@ -60,13 +62,18 @@ public final class RegistryJpaIO {
|
||||
|
||||
private RegistryJpaIO() {}
|
||||
|
||||
public static <R> Read<R, R> read(QueryComposerFactory<R> queryFactory) {
|
||||
return Read.<R, R>builder().queryFactory(queryFactory).build();
|
||||
public static <R> Read<R, R> read(CriteriaQuerySupplier<R> query) {
|
||||
return read(query, x -> x);
|
||||
}
|
||||
|
||||
public static <R, T> Read<R, T> read(
|
||||
QueryComposerFactory<R> queryFactory, SerializableFunction<R, T> resultMapper) {
|
||||
return Read.<R, T>builder().queryFactory(queryFactory).resultMapper(resultMapper).build();
|
||||
CriteriaQuerySupplier<R> query, SerializableFunction<R, T> resultMapper) {
|
||||
return Read.<R, T>builder().criteriaQuery(query).resultMapper(resultMapper).build();
|
||||
}
|
||||
|
||||
public static <R, T> Read<R, T> read(
|
||||
String sql, boolean nativeQuery, SerializableFunction<R, T> resultMapper) {
|
||||
return read(sql, null, nativeQuery, resultMapper);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -74,8 +81,39 @@ public final class RegistryJpaIO {
|
||||
*
|
||||
* <p>User should take care to prevent sql-injection attacks.
|
||||
*/
|
||||
public static <R, T> Read<R, T> read(String jpql, SerializableFunction<R, T> resultMapper) {
|
||||
return Read.<R, T>builder().jpqlQueryFactory(jpql).resultMapper(resultMapper).build();
|
||||
public static <R, T> Read<R, T> read(
|
||||
String sql,
|
||||
@Nullable Map<String, Object> parameter,
|
||||
boolean nativeQuery,
|
||||
SerializableFunction<R, T> resultMapper) {
|
||||
Read.Builder<R, T> builder = Read.builder();
|
||||
if (nativeQuery) {
|
||||
builder.nativeQuery(sql, parameter);
|
||||
} else {
|
||||
builder.jpqlQuery(sql, parameter);
|
||||
}
|
||||
return builder.resultMapper(resultMapper).build();
|
||||
}
|
||||
|
||||
public static <R, T> Read<R, T> read(
|
||||
String jpql, Class<R> clazz, SerializableFunction<R, T> resultMapper) {
|
||||
return read(jpql, null, clazz, resultMapper);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link Read} connector based on the given {@code jpql} typed query string.
|
||||
*
|
||||
* <p>User should take care to prevent sql-injection attacks.
|
||||
*/
|
||||
public static <R, T> Read<R, T> read(
|
||||
String jpql,
|
||||
@Nullable Map<String, Object> parameter,
|
||||
Class<R> clazz,
|
||||
SerializableFunction<R, T> resultMapper) {
|
||||
return Read.<R, T>builder()
|
||||
.jpqlQuery(jpql, clazz, parameter)
|
||||
.resultMapper(resultMapper)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static <T> Write<T> write() {
|
||||
@@ -94,7 +132,7 @@ public final class RegistryJpaIO {
|
||||
|
||||
abstract String name();
|
||||
|
||||
abstract RegistryQueryFactory<R> queryFactory();
|
||||
abstract RegistryQuery<R> query();
|
||||
|
||||
abstract SerializableFunction<R, T> resultMapper();
|
||||
|
||||
@@ -107,9 +145,7 @@ public final class RegistryJpaIO {
|
||||
public PCollection<T> expand(PBegin input) {
|
||||
return input
|
||||
.apply("Starting " + name(), Create.of((Void) null))
|
||||
.apply(
|
||||
"Run query for " + name(),
|
||||
ParDo.of(new QueryRunner<>(queryFactory(), resultMapper())))
|
||||
.apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
|
||||
.setCoder(coder())
|
||||
.apply("Reshuffle", Reshuffle.viaRandomKey());
|
||||
}
|
||||
@@ -127,9 +163,8 @@ public final class RegistryJpaIO {
|
||||
}
|
||||
|
||||
static <R, T> Builder<R, T> builder() {
|
||||
return new AutoValue_RegistryJpaIO_Read.Builder()
|
||||
return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
|
||||
.name(DEFAULT_NAME)
|
||||
.resultMapper(x -> x)
|
||||
.coder(SerializableCoder.of(Serializable.class));
|
||||
}
|
||||
|
||||
@@ -138,7 +173,7 @@ public final class RegistryJpaIO {
|
||||
|
||||
abstract Builder<R, T> name(String name);
|
||||
|
||||
abstract Builder<R, T> queryFactory(RegistryQueryFactory<R> queryFactory);
|
||||
abstract Builder<R, T> query(RegistryQuery<R> query);
|
||||
|
||||
abstract Builder<R, T> resultMapper(SerializableFunction<R, T> mapper);
|
||||
|
||||
@@ -146,21 +181,29 @@ public final class RegistryJpaIO {
|
||||
|
||||
abstract Read<R, T> build();
|
||||
|
||||
Builder<R, T> queryFactory(QueryComposerFactory<R> queryFactory) {
|
||||
return queryFactory(RegistryQuery.createQueryFactory(queryFactory));
|
||||
Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
|
||||
return query(RegistryQuery.createQuery(criteriaQuery));
|
||||
}
|
||||
|
||||
Builder<R, T> jpqlQueryFactory(String jpql) {
|
||||
return queryFactory(RegistryQuery.createQueryFactory(jpql));
|
||||
Builder<R, T> nativeQuery(String sql, Map<String, Object> parameters) {
|
||||
return query(RegistryQuery.createQuery(sql, parameters, true));
|
||||
}
|
||||
|
||||
Builder<R, T> jpqlQuery(String jpql, Map<String, Object> parameters) {
|
||||
return query(RegistryQuery.createQuery(jpql, parameters, false));
|
||||
}
|
||||
|
||||
Builder<R, T> jpqlQuery(String jpql, Class<R> clazz, Map<String, Object> parameters) {
|
||||
return query(RegistryQuery.createQuery(jpql, parameters, clazz));
|
||||
}
|
||||
}
|
||||
|
||||
static class QueryRunner<R, T> extends DoFn<Void, T> {
|
||||
private final RegistryQueryFactory<R> queryFactory;
|
||||
private final RegistryQuery<R> query;
|
||||
private final SerializableFunction<R, T> resultMapper;
|
||||
|
||||
QueryRunner(RegistryQueryFactory<R> queryFactory, SerializableFunction<R, T> resultMapper) {
|
||||
this.queryFactory = queryFactory;
|
||||
QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
|
||||
this.query = query;
|
||||
this.resultMapper = resultMapper;
|
||||
}
|
||||
|
||||
@@ -172,10 +215,7 @@ public final class RegistryJpaIO {
|
||||
// TODO(b/187210388): JpaTransactionManager should support non-transactional query.
|
||||
jpaTm()
|
||||
.transactNoRetry(
|
||||
() ->
|
||||
queryFactory.apply(jpaTm()).stream()
|
||||
.map(resultMapper::apply)
|
||||
.forEach(outputReceiver::output));
|
||||
() -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -319,17 +359,17 @@ public final class RegistryJpaIO {
|
||||
@ProcessElement
|
||||
public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
|
||||
try (AppEngineEnvironment env = new AppEngineEnvironment()) {
|
||||
ImmutableList<Object> ofyEntities =
|
||||
ImmutableList<Object> entities =
|
||||
Streams.stream(kv.getValue())
|
||||
.map(this.jpaConverter::apply)
|
||||
// TODO(b/177340730): post migration delete the line below.
|
||||
.filter(Objects::nonNull)
|
||||
.collect(ImmutableList.toImmutableList());
|
||||
try {
|
||||
jpaTm().transact(() -> jpaTm().putAll(ofyEntities));
|
||||
counter.inc(ofyEntities.size());
|
||||
jpaTm().transact(() -> jpaTm().putAll(entities));
|
||||
counter.inc(entities.size());
|
||||
} catch (RuntimeException e) {
|
||||
processSingly(ofyEntities);
|
||||
processSingly(entities);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -338,19 +378,22 @@ public final class RegistryJpaIO {
|
||||
* Writes entities in a failed batch one by one to identify the first bad entity and throws a
|
||||
* {@link RuntimeException} on it.
|
||||
*/
|
||||
private void processSingly(ImmutableList<Object> ofyEntities) {
|
||||
for (Object ofyEntity : ofyEntities) {
|
||||
private void processSingly(ImmutableList<Object> entities) {
|
||||
for (Object entity : entities) {
|
||||
try {
|
||||
jpaTm().transact(() -> jpaTm().put(ofyEntity));
|
||||
jpaTm().transact(() -> jpaTm().put(entity));
|
||||
counter.inc();
|
||||
} catch (RuntimeException e) {
|
||||
throw new RuntimeException(toOfyKey(ofyEntity).toString(), e);
|
||||
throw new RuntimeException(toEntityKeyString(entity), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private com.googlecode.objectify.Key<?> toOfyKey(Object ofyEntity) {
|
||||
return com.googlecode.objectify.Key.create(ofyEntity);
|
||||
private String toEntityKeyString(Object entity) {
|
||||
if (entity instanceof SqlEntity) {
|
||||
return ((SqlEntity) entity).getPrimaryKeyString();
|
||||
}
|
||||
return "Non-SqlEntity: " + String.valueOf(entity);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,48 +14,75 @@
|
||||
|
||||
package google.registry.beam.common;
|
||||
|
||||
import google.registry.persistence.transaction.JpaTransactionManager;
|
||||
import google.registry.persistence.transaction.QueryComposer;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Stream;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.persistence.EntityManager;
|
||||
import javax.persistence.Query;
|
||||
import org.apache.beam.sdk.transforms.SerializableFunction;
|
||||
import javax.persistence.TypedQuery;
|
||||
import javax.persistence.criteria.CriteriaQuery;
|
||||
|
||||
/** Interface for query instances used by {@link RegistryJpaIO.Read}. */
|
||||
public interface RegistryQuery<T> {
|
||||
public interface RegistryQuery<T> extends Serializable {
|
||||
Stream<T> stream();
|
||||
|
||||
/** Factory for {@link RegistryQuery}. */
|
||||
interface RegistryQueryFactory<T>
|
||||
extends SerializableFunction<JpaTransactionManager, RegistryQuery<T>> {}
|
||||
|
||||
// TODO(mmuller): Consider detached JpaQueryComposer that works with any JpaTransactionManager
|
||||
// instance, i.e., change composer.buildQuery() to composer.buildQuery(JpaTransactionManager).
|
||||
// This way QueryComposer becomes reusable and serializable (at least with Hibernate), and this
|
||||
// interface would no longer be necessary.
|
||||
interface QueryComposerFactory<T>
|
||||
extends SerializableFunction<JpaTransactionManager, QueryComposer<T>> {}
|
||||
interface CriteriaQuerySupplier<T> extends Supplier<CriteriaQuery<T>>, Serializable {}
|
||||
|
||||
/**
|
||||
* Returns a {@link RegistryQueryFactory} that creates a JPQL query from constant text.
|
||||
* Returns a {@link RegistryQuery} that creates a string query from constant text.
|
||||
*
|
||||
* @param nativeQuery whether the given string is to be interpreted as a native query or JPQL.
|
||||
* @param parameters parameters to be substituted in the query.
|
||||
* @param <T> Type of each row in the result set, {@link Object} in single-select queries, and
|
||||
* {@code Object[]} in multi-select queries.
|
||||
*/
|
||||
@SuppressWarnings("unchecked") // query.getResultStream: jpa api uses raw type
|
||||
static <T> RegistryQueryFactory<T> createQueryFactory(String jpql) {
|
||||
return (JpaTransactionManager jpa) ->
|
||||
() -> {
|
||||
EntityManager entityManager = jpa.getEntityManager();
|
||||
Query query = entityManager.createQuery(jpql);
|
||||
return query.getResultStream().map(e -> detach(entityManager, e));
|
||||
};
|
||||
static <T> RegistryQuery<T> createQuery(
|
||||
String sql, @Nullable Map<String, Object> parameters, boolean nativeQuery) {
|
||||
return () -> {
|
||||
EntityManager entityManager = jpaTm().getEntityManager();
|
||||
Query query =
|
||||
nativeQuery ? entityManager.createNativeQuery(sql) : entityManager.createQuery(sql);
|
||||
if (parameters != null) {
|
||||
parameters.forEach(query::setParameter);
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
Stream<T> resultStream = query.getResultStream();
|
||||
return nativeQuery ? resultStream : resultStream.map(e -> detach(entityManager, e));
|
||||
};
|
||||
}
|
||||
|
||||
static <T> RegistryQueryFactory<T> createQueryFactory(
|
||||
QueryComposerFactory<T> queryComposerFactory) {
|
||||
return (JpaTransactionManager jpa) ->
|
||||
() -> queryComposerFactory.apply(jpa).withAutoDetachOnLoad(true).stream();
|
||||
/**
|
||||
* Returns a {@link RegistryQuery} that creates a typed JPQL query from constant text.
|
||||
*
|
||||
* @param parameters parameters to be substituted in the query.
|
||||
* @param <T> Type of each row in the result set.
|
||||
*/
|
||||
static <T> RegistryQuery<T> createQuery(
|
||||
String jpql, @Nullable Map<String, Object> parameters, Class<T> clazz) {
|
||||
return () -> {
|
||||
TypedQuery<T> query = jpaTm().query(jpql, clazz);
|
||||
if (parameters != null) {
|
||||
parameters.forEach(query::setParameter);
|
||||
}
|
||||
return query.getResultStream();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link RegistryQuery} from a {@link CriteriaQuery} supplier.
|
||||
*
|
||||
* <p>A serializable supplier is needed in because {@link CriteriaQuery} itself must be created
|
||||
* within a transaction, and we are not in a transaction yet when this function is called to set
|
||||
* up the pipeline.
|
||||
*
|
||||
* @param <T> Type of each row in the result set.
|
||||
*/
|
||||
static <T> RegistryQuery<T> createQuery(CriteriaQuerySupplier<T> criteriaQuery) {
|
||||
return () -> jpaTm().query(criteriaQuery.get()).getResultStream();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.initsql;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
|
||||
import com.google.api.services.cloudkms.v1.model.DecryptRequest;
|
||||
import com.google.common.base.Strings;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.keyring.kms.KmsConnection;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Base64;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Decrypts data using Cloud KMS, with the same crypto key with which Cloud SQL credential files on
|
||||
* GCS was encrypted. See {@link BackupPaths#getCloudSQLCredentialFilePatterns} for more
|
||||
* information.
|
||||
*/
|
||||
public class CloudSqlCredentialDecryptor {
|
||||
|
||||
private static final String CRYPTO_KEY_NAME = "nomulus-tool-key";
|
||||
private final KmsConnection kmsConnection;
|
||||
|
||||
@Inject
|
||||
CloudSqlCredentialDecryptor(@Config("beamKmsConnection") KmsConnection kmsConnection) {
|
||||
this.kmsConnection = kmsConnection;
|
||||
}
|
||||
|
||||
public String decrypt(String data) {
|
||||
checkArgument(!Strings.isNullOrEmpty(data), "Null or empty data.");
|
||||
byte[] ciphertext = Base64.getDecoder().decode(data);
|
||||
// Re-encode for Cloud KMS JSON REST API, invoked through kmsConnection.
|
||||
String urlSafeCipherText = new DecryptRequest().encodeCiphertext(ciphertext).getCiphertext();
|
||||
return new String(
|
||||
kmsConnection.decrypt(CRYPTO_KEY_NAME, urlSafeCipherText), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
@@ -32,8 +32,8 @@ import google.registry.model.host.HostResource;
|
||||
import google.registry.model.poll.PollMessage;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarContact;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import java.io.Serializable;
|
||||
import java.util.Collection;
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.initsql;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Information needed to connect to a database, including JDBC URL, user name, password, and in the
|
||||
* case of Cloud SQL, the database instance's name.
|
||||
*/
|
||||
@AutoValue
|
||||
abstract class SqlAccessInfo {
|
||||
|
||||
abstract String jdbcUrl();
|
||||
|
||||
abstract String user();
|
||||
|
||||
abstract String password();
|
||||
|
||||
abstract Optional<String> cloudSqlInstanceName();
|
||||
|
||||
public static SqlAccessInfo createCloudSqlAccessInfo(
|
||||
String sqlInstanceName, String username, String password) {
|
||||
return new AutoValue_SqlAccessInfo(
|
||||
"jdbc:postgresql://google/postgres", username, password, Optional.of(sqlInstanceName));
|
||||
}
|
||||
|
||||
public static SqlAccessInfo createLocalSqlAccessInfo(
|
||||
String jdbcUrl, String username, String password) {
|
||||
return new AutoValue_SqlAccessInfo(jdbcUrl, username, password, Optional.empty());
|
||||
}
|
||||
}
|
||||
@@ -39,9 +39,9 @@ import google.registry.backup.VersionedEntity;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.replay.DatastoreAndSqlEntity;
|
||||
import google.registry.model.replay.SqlEntity;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.schema.replay.DatastoreAndSqlEntity;
|
||||
import google.registry.schema.replay.SqlEntity;
|
||||
import google.registry.tools.LevelDbLogReader;
|
||||
import java.io.Serializable;
|
||||
import java.util.Collection;
|
||||
|
||||
@@ -14,18 +14,29 @@
|
||||
|
||||
package google.registry.beam.invoicing;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.beam.BeamUtils.getQueryFromFile;
|
||||
import static org.apache.beam.sdk.values.TypeDescriptors.strings;
|
||||
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.beam.common.RegistryJpaIO.Read;
|
||||
import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey;
|
||||
import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.reporting.billing.BillingModule;
|
||||
import google.registry.util.DateTimeUtils;
|
||||
import google.registry.util.DomainNameUtils;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.LocalTime;
|
||||
import java.time.YearMonth;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Pattern;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.PipelineResult;
|
||||
import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
@@ -58,6 +69,9 @@ public class InvoicingPipeline implements Serializable {
|
||||
private static final DateTimeFormatter TIMESTAMP_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS");
|
||||
|
||||
private static final Pattern SQL_COMMENT_REGEX =
|
||||
Pattern.compile("^\\s*--.*\\n", Pattern.MULTILINE);
|
||||
|
||||
private final InvoicingPipelineOptions options;
|
||||
|
||||
InvoicingPipeline(InvoicingPipelineOptions options) {
|
||||
@@ -71,21 +85,60 @@ public class InvoicingPipeline implements Serializable {
|
||||
}
|
||||
|
||||
void setupPipeline(Pipeline pipeline) {
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
PCollection<BillingEvent> billingEvents =
|
||||
pipeline.apply(
|
||||
"Read BillingEvents from Bigquery",
|
||||
BigQueryIO.read(BillingEvent::parseFromRecord)
|
||||
.fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
|
||||
.withCoder(SerializableCoder.of(BillingEvent.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
options.getDatabase().equals("DATASTORE")
|
||||
? readFromBigQuery(options, pipeline)
|
||||
: readFromCloudSql(options, pipeline);
|
||||
|
||||
saveInvoiceCsv(billingEvents, options);
|
||||
|
||||
saveDetailedCsv(billingEvents, options);
|
||||
}
|
||||
|
||||
static PCollection<BillingEvent> readFromBigQuery(
|
||||
InvoicingPipelineOptions options, Pipeline pipeline) {
|
||||
return pipeline.apply(
|
||||
"Read BillingEvents from Bigquery",
|
||||
BigQueryIO.read(BillingEvent::parseFromRecord)
|
||||
.fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
|
||||
.withCoder(SerializableCoder.of(BillingEvent.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
}
|
||||
|
||||
static PCollection<BillingEvent> readFromCloudSql(
|
||||
InvoicingPipelineOptions options, Pipeline pipeline) {
|
||||
Read<Object[], BillingEvent> read =
|
||||
RegistryJpaIO.read(
|
||||
makeCloudSqlQuery(options.getYearMonth()), false, InvoicingPipeline::parseRow);
|
||||
|
||||
return pipeline.apply("Read BillingEvents from Cloud SQL", read);
|
||||
}
|
||||
|
||||
private static BillingEvent parseRow(Object[] row) {
|
||||
google.registry.model.billing.BillingEvent.OneTime oneTime =
|
||||
(google.registry.model.billing.BillingEvent.OneTime) row[0];
|
||||
Registrar registrar = (Registrar) row[1];
|
||||
return BillingEvent.create(
|
||||
oneTime.getId(),
|
||||
DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
|
||||
DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
|
||||
registrar.getClientId(),
|
||||
registrar.getBillingIdentifier().toString(),
|
||||
registrar.getPoNumber().orElse(""),
|
||||
DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),
|
||||
oneTime.getReason().toString(),
|
||||
oneTime.getTargetId(),
|
||||
oneTime.getDomainRepoId(),
|
||||
Optional.ofNullable(oneTime.getPeriodYears()).orElse(0),
|
||||
oneTime.getCost().getCurrencyUnit().toString(),
|
||||
oneTime.getCost().getAmount().doubleValue(),
|
||||
String.join(
|
||||
" ", oneTime.getFlags().stream().map(Flag::toString).collect(toImmutableSet())));
|
||||
}
|
||||
|
||||
/** Transform that converts a {@code BillingEvent} into an invoice CSV row. */
|
||||
private static class GenerateInvoiceRows
|
||||
extends PTransform<PCollection<BillingEvent>, PCollection<String>> {
|
||||
@@ -171,6 +224,21 @@ public class InvoicingPipeline implements Serializable {
|
||||
.build();
|
||||
}
|
||||
|
||||
/** Create the Cloud SQL query for a given yearMonth at runtime. */
|
||||
static String makeCloudSqlQuery(String yearMonth) {
|
||||
YearMonth endMonth = YearMonth.parse(yearMonth).plusMonths(1);
|
||||
String queryWithComments =
|
||||
SqlTemplate.create(
|
||||
getQueryFromFile(InvoicingPipeline.class, "cloud_sql_billing_events.sql"))
|
||||
.put("FIRST_TIMESTAMP_OF_MONTH", yearMonth.concat("-01"))
|
||||
.put(
|
||||
"LAST_TIMESTAMP_OF_MONTH",
|
||||
String.format("%d-%d-01", endMonth.getYear(), endMonth.getMonthValue()))
|
||||
.build();
|
||||
// Remove the comments from the query string
|
||||
return SQL_COMMENT_REGEX.matcher(queryWithComments).replaceAll("");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
PipelineOptionsFactory.register(InvoicingPipelineOptions.class);
|
||||
InvoicingPipelineOptions options =
|
||||
|
||||
@@ -30,6 +30,11 @@ public interface InvoicingPipelineOptions extends RegistryPipelineOptions {
|
||||
|
||||
void setInvoiceFilePrefix(String value);
|
||||
|
||||
@Description("The database to read data from.")
|
||||
String getDatabase();
|
||||
|
||||
void setDatabase(String value);
|
||||
|
||||
@Description("The GCS bucket URL for invoices and detailed reports to be uploaded.")
|
||||
String getBillingBucketUrl();
|
||||
|
||||
|
||||
273
core/src/main/java/google/registry/beam/rde/RdeIO.java
Normal file
273
core/src/main/java/google/registry/beam/rde/RdeIO.java
Normal file
@@ -0,0 +1,273 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.rde;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.rde.RdeMode;
|
||||
import google.registry.model.rde.RdeNamingUtils;
|
||||
import google.registry.model.rde.RdeRevision;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.Ghostryde;
|
||||
import google.registry.rde.PendingDeposit;
|
||||
import google.registry.rde.RdeCounter;
|
||||
import google.registry.rde.RdeMarshaller;
|
||||
import google.registry.rde.RdeResourceType;
|
||||
import google.registry.rde.RdeUtil;
|
||||
import google.registry.tldconfig.idn.IdnTableEnum;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeader;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
|
||||
import google.registry.xml.ValidationMode;
|
||||
import google.registry.xml.XmlException;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.security.Security;
|
||||
import java.util.Optional;
|
||||
import org.apache.beam.sdk.options.PipelineOptions;
|
||||
import org.apache.beam.sdk.transforms.DoFn;
|
||||
import org.apache.beam.sdk.transforms.PTransform;
|
||||
import org.apache.beam.sdk.transforms.ParDo;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.apache.beam.sdk.values.PDone;
|
||||
import org.bouncycastle.jce.provider.BouncyCastleProvider;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
public class RdeIO {
|
||||
|
||||
@AutoValue
|
||||
abstract static class Write
|
||||
extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
|
||||
|
||||
abstract GcsUtils gcsUtils();
|
||||
|
||||
abstract String rdeBucket();
|
||||
|
||||
// It's OK to return a primitive array because we are only using it to construct the
|
||||
// PGPPublicKey, which is not serializable.
|
||||
@SuppressWarnings("mutable")
|
||||
abstract byte[] stagingKeyBytes();
|
||||
|
||||
abstract ValidationMode validationMode();
|
||||
|
||||
static Builder builder() {
|
||||
return new AutoValue_RdeIO_Write.Builder();
|
||||
}
|
||||
|
||||
@AutoValue.Builder
|
||||
abstract static class Builder {
|
||||
abstract Builder setGcsUtils(GcsUtils gcsUtils);
|
||||
|
||||
abstract Builder setRdeBucket(String value);
|
||||
|
||||
abstract Builder setStagingKeyBytes(byte[] value);
|
||||
|
||||
abstract Builder setValidationMode(ValidationMode value);
|
||||
|
||||
abstract Write build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PDone expand(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
|
||||
input
|
||||
.apply(
|
||||
"Write to GCS",
|
||||
ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
|
||||
.apply("Update cursors", ParDo.of(new CursorUpdater()));
|
||||
return PDone.in(input.getPipeline());
|
||||
}
|
||||
}
|
||||
|
||||
private static class RdeWriter
|
||||
extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private final GcsUtils gcsUtils;
|
||||
private final String rdeBucket;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final RdeMarshaller marshaller;
|
||||
|
||||
protected RdeWriter(
|
||||
GcsUtils gcsUtils,
|
||||
String rdeBucket,
|
||||
byte[] stagingKeyBytes,
|
||||
ValidationMode validationMode) {
|
||||
this.gcsUtils = gcsUtils;
|
||||
this.rdeBucket = rdeBucket;
|
||||
this.stagingKeyBytes = stagingKeyBytes;
|
||||
this.marshaller = new RdeMarshaller(validationMode);
|
||||
}
|
||||
|
||||
@Setup
|
||||
public void setup() {
|
||||
Security.addProvider(new BouncyCastleProvider());
|
||||
}
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(
|
||||
@Element KV<PendingDeposit, Iterable<DepositFragment>> kv,
|
||||
PipelineOptions options,
|
||||
OutputReceiver<KV<PendingDeposit, Integer>> outputReceiver) {
|
||||
PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
|
||||
PendingDeposit key = kv.getKey();
|
||||
Iterable<DepositFragment> fragments = kv.getValue();
|
||||
RdeCounter counter = new RdeCounter();
|
||||
|
||||
// Determine some basic things about the deposit.
|
||||
final RdeMode mode = key.mode();
|
||||
final String tld = key.tld();
|
||||
final DateTime watermark = key.watermark();
|
||||
final int revision =
|
||||
Optional.ofNullable(key.revision())
|
||||
.orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
|
||||
String id = RdeUtil.timestampToId(watermark);
|
||||
String prefix = options.getJobName();
|
||||
String basename = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
|
||||
if (key.manual()) {
|
||||
checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
|
||||
prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
|
||||
} else {
|
||||
prefix = prefix + "/" + basename;
|
||||
}
|
||||
BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
|
||||
// This file will contain the byte length (ASCII) of the raw unencrypted XML.
|
||||
//
|
||||
// This is necessary because RdeUploadAction creates a tar file which requires that the length
|
||||
// be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
|
||||
// length, so we just save it separately.
|
||||
BlobId xmlLengthFilename = BlobId.of(rdeBucket, prefix + ".xml.length");
|
||||
BlobId reportFilename = BlobId.of(rdeBucket, prefix + "-report.xml.ghostryde");
|
||||
|
||||
// These variables will be populated as we write the deposit XML and used for other files.
|
||||
boolean failed = false;
|
||||
XjcRdeHeader header;
|
||||
|
||||
// Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
|
||||
|
||||
logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
|
||||
Writer output = new OutputStreamWriter(ghostrydeEncoder, UTF_8)) {
|
||||
|
||||
// Output the top portion of the XML document.
|
||||
output.write(marshaller.makeHeader(id, watermark, RdeResourceType.getUris(mode), revision));
|
||||
|
||||
// Output XML fragments while counting them.
|
||||
for (DepositFragment fragment : fragments) {
|
||||
if (!fragment.xml().isEmpty()) {
|
||||
output.write(fragment.xml());
|
||||
counter.increment(fragment.type());
|
||||
}
|
||||
if (!fragment.error().isEmpty()) {
|
||||
failed = true;
|
||||
logger.atSevere().log("Fragment error: %s", fragment.error());
|
||||
}
|
||||
}
|
||||
|
||||
// Don't write the IDN elements for BRDA.
|
||||
if (mode == RdeMode.FULL) {
|
||||
for (IdnTableEnum idn : IdnTableEnum.values()) {
|
||||
output.write(marshaller.marshalIdn(idn.getTable()));
|
||||
counter.increment(RdeResourceType.IDN);
|
||||
}
|
||||
}
|
||||
|
||||
// Output XML that says how many resources were emitted.
|
||||
header = counter.makeHeader(tld, mode);
|
||||
output.write(marshaller.marshalOrDie(new XjcRdeHeaderElement(header)));
|
||||
|
||||
// Output the bottom of the XML document.
|
||||
output.write(marshaller.makeFooter());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
// If an entity was broken, abort after writing as much logs/deposit data as possible.
|
||||
verify(!failed, "RDE staging failed for TLD %s", tld);
|
||||
|
||||
// Write a tiny XML file to GCS containing some information about the deposit.
|
||||
//
|
||||
// This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
|
||||
if (mode == RdeMode.FULL) {
|
||||
logger.atInfo().log("Writing %s", reportFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
|
||||
counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
|
||||
} catch (IOException | XmlException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
// Now that we're done, output roll the cursor forward.
|
||||
if (key.manual()) {
|
||||
logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task");
|
||||
} else {
|
||||
outputReceiver.output(KV.of(key, revision));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(@Element KV<PendingDeposit, Integer> input) {
|
||||
tm().transact(
|
||||
() -> {
|
||||
PendingDeposit key = input.getKey();
|
||||
int revision = input.getValue();
|
||||
Registry registry = Registry.get(key.tld());
|
||||
Optional<Cursor> cursor =
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
tm().loadByKeyIfPresent(
|
||||
Cursor.createVKey(key.cursor(), registry.getTldStr())));
|
||||
DateTime position = getCursorTimeOrStartOfTime(cursor);
|
||||
checkState(key.interval() != null, "Interval must be present");
|
||||
DateTime newPosition = key.watermark().plus(key.interval());
|
||||
if (!position.isBefore(newPosition)) {
|
||||
logger.atWarning().log("Cursor has already been rolled forward.");
|
||||
return;
|
||||
}
|
||||
verify(
|
||||
position.equals(key.watermark()),
|
||||
"Partial ordering of RDE deposits broken: %s %s",
|
||||
position,
|
||||
key);
|
||||
tm().put(Cursor.create(key.cursor(), newPosition, registry));
|
||||
logger.atInfo().log(
|
||||
"Rolled forward %s on %s cursor to %s", key.cursor(), key.tld(), newPosition);
|
||||
RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
303
core/src/main/java/google/registry/beam/rde/RdePipeline.java
Normal file
303
core/src/main/java/google/registry/beam/rde/RdePipeline.java
Normal file
@@ -0,0 +1,303 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.rde;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import dagger.BindsInstance;
|
||||
import dagger.Component;
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.config.CredentialModule;
|
||||
import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.rde.RdeMode;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.Registrar.Type;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.PendingDeposit;
|
||||
import google.registry.rde.PendingDeposit.PendingDepositCoder;
|
||||
import google.registry.rde.RdeFragmenter;
|
||||
import google.registry.rde.RdeMarshaller;
|
||||
import google.registry.xml.ValidationMode;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Singleton;
|
||||
import javax.persistence.Entity;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.PipelineResult;
|
||||
import org.apache.beam.sdk.coders.KvCoder;
|
||||
import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.transforms.FlatMapElements;
|
||||
import org.apache.beam.sdk.transforms.Flatten;
|
||||
import org.apache.beam.sdk.transforms.GroupByKey;
|
||||
import org.apache.beam.sdk.transforms.Reshuffle;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.apache.beam.sdk.values.PCollectionList;
|
||||
import org.apache.beam.sdk.values.TypeDescriptor;
|
||||
import org.apache.beam.sdk.values.TypeDescriptors;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
|
||||
*
|
||||
* <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
|
||||
*
|
||||
* <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
|
||||
*
|
||||
* @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
|
||||
* Flex Templates</a>
|
||||
*/
|
||||
@Singleton
|
||||
public class RdePipeline implements Serializable {
|
||||
|
||||
private final transient RdePipelineOptions options;
|
||||
private final ValidationMode mode;
|
||||
private final ImmutableSetMultimap<String, PendingDeposit> pendings;
|
||||
private final String rdeBucket;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
// Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
|
||||
// if sneaks into production we would get an extra signal.
|
||||
private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
|
||||
Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
|
||||
|
||||
private static final String EPP_RESOURCE_QUERY =
|
||||
"SELECT id FROM %entity% "
|
||||
+ "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
|
||||
+ "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
|
||||
+ "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
|
||||
|
||||
public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
|
||||
return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
|
||||
+ (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
|
||||
}
|
||||
|
||||
@Inject
|
||||
RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
|
||||
this.options = options;
|
||||
this.mode = ValidationMode.valueOf(options.getValidationMode());
|
||||
this.pendings = decodePendings(options.getPendings());
|
||||
this.rdeBucket = options.getGcsBucket();
|
||||
this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
PipelineResult run() {
|
||||
Pipeline pipeline = Pipeline.create(options);
|
||||
PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> fragments =
|
||||
createFragments(pipeline);
|
||||
persistData(fragments);
|
||||
return pipeline.run();
|
||||
}
|
||||
|
||||
PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
|
||||
return PCollectionList.of(processRegistrars(pipeline))
|
||||
.and(processNonRegistrarEntities(pipeline, DomainBase.class))
|
||||
.and(processNonRegistrarEntities(pipeline, ContactResource.class))
|
||||
.and(processNonRegistrarEntities(pipeline, HostResource.class))
|
||||
.apply(Flatten.pCollections())
|
||||
.setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
|
||||
.apply("Group by PendingDeposit", GroupByKey.create());
|
||||
}
|
||||
|
||||
void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
|
||||
input.apply(
|
||||
"Write to GCS and update cursors",
|
||||
RdeIO.Write.builder()
|
||||
.setRdeBucket(rdeBucket)
|
||||
.setGcsUtils(gcsUtils)
|
||||
.setValidationMode(mode)
|
||||
.setStagingKeyBytes(stagingKeyBytes)
|
||||
.build());
|
||||
}
|
||||
|
||||
PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
|
||||
return pipeline
|
||||
.apply(
|
||||
"Read all production Registrar entities",
|
||||
RegistryJpaIO.read(
|
||||
"SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
|
||||
ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
|
||||
String.class,
|
||||
// TODO: consider adding coders for entities and pass them directly instead of using
|
||||
// VKeys.
|
||||
id -> VKey.createSql(Registrar.class, id)))
|
||||
.apply(
|
||||
"Marshal Registrar into DepositFragment",
|
||||
FlatMapElements.into(
|
||||
TypeDescriptors.kvs(
|
||||
TypeDescriptor.of(PendingDeposit.class),
|
||||
TypeDescriptor.of(DepositFragment.class)))
|
||||
.via(
|
||||
(VKey<Registrar> key) -> {
|
||||
Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
|
||||
DepositFragment fragment =
|
||||
new RdeMarshaller(mode).marshalRegistrar(registrar);
|
||||
return pendings.values().stream()
|
||||
.map(pending -> KV.of(pending, fragment))
|
||||
.collect(toImmutableSet());
|
||||
}));
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation") // Reshuffle is still recommended by Dataflow.
|
||||
<T extends EppResource>
|
||||
PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
|
||||
Pipeline pipeline, Class<T> clazz) {
|
||||
return createInputs(pipeline, clazz)
|
||||
.apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
|
||||
.setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
|
||||
.apply(
|
||||
"Reshuffle KV<PendingDeposit, DepositFragment> of "
|
||||
+ clazz.getSimpleName()
|
||||
+ " to prevent fusion",
|
||||
Reshuffle.of());
|
||||
}
|
||||
|
||||
<T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
|
||||
return pipeline.apply(
|
||||
"Read all production " + clazz.getSimpleName() + " entities",
|
||||
RegistryJpaIO.read(
|
||||
createEppResourceQuery(clazz),
|
||||
clazz.equals(DomainBase.class)
|
||||
? ImmutableMap.of("tlds", pendings.keySet())
|
||||
: ImmutableMap.of(),
|
||||
String.class,
|
||||
// TODO: consider adding coders for entities and pass them directly instead of using
|
||||
// VKeys.
|
||||
x -> VKey.create(clazz, x)));
|
||||
}
|
||||
|
||||
<T extends EppResource>
|
||||
FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
|
||||
return FlatMapElements.into(
|
||||
TypeDescriptors.kvs(
|
||||
TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
|
||||
.via(
|
||||
(VKey<T> key) -> {
|
||||
T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
|
||||
// The set of all TLDs to which this resource should be emitted.
|
||||
ImmutableSet<String> tlds =
|
||||
clazz.equals(DomainBase.class)
|
||||
? ImmutableSet.of(((DomainBase) resource).getTld())
|
||||
: pendings.keySet();
|
||||
// Get the set of all point-in-time watermarks we need, to minimize rewinding.
|
||||
ImmutableSet<DateTime> dates =
|
||||
tlds.stream()
|
||||
.map(pendings::get)
|
||||
.flatMap(ImmutableSet::stream)
|
||||
.map(PendingDeposit::watermark)
|
||||
.collect(toImmutableSet());
|
||||
// Launch asynchronous fetches of point-in-time representations of resource.
|
||||
ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
|
||||
ImmutableMap.copyOf(
|
||||
Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
|
||||
// Convert resource to an XML fragment for each watermark/mode pair lazily and cache
|
||||
// the result.
|
||||
RdeFragmenter fragmenter =
|
||||
new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
|
||||
List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
|
||||
for (String tld : tlds) {
|
||||
for (PendingDeposit pending : pendings.get(tld)) {
|
||||
// Hosts and contacts don't get included in BRDA deposits.
|
||||
if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
|
||||
continue;
|
||||
}
|
||||
Optional<DepositFragment> fragment =
|
||||
fragmenter.marshal(pending.watermark(), pending.mode());
|
||||
fragment.ifPresent(
|
||||
depositFragment -> results.add(KV.of(pending, depositFragment)));
|
||||
}
|
||||
}
|
||||
return results;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
|
||||
* the original TLD to pending deposit map.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
|
||||
try (ObjectInputStream ois =
|
||||
new ObjectInputStream(
|
||||
new ByteArrayInputStream(
|
||||
BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
|
||||
return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
|
||||
} catch (IOException | ClassNotFoundException e) {
|
||||
throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
|
||||
* worker by the pipeline launcher as a pipeline option.
|
||||
*/
|
||||
static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
|
||||
throws IOException {
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
ObjectOutputStream oos = new ObjectOutputStream(baos);
|
||||
oos.writeObject(pendings);
|
||||
oos.flush();
|
||||
return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException, ClassNotFoundException {
|
||||
PipelineOptionsFactory.register(RdePipelineOptions.class);
|
||||
RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
|
||||
}
|
||||
|
||||
@Singleton
|
||||
@Component(modules = {CredentialModule.class, ConfigModule.class})
|
||||
interface RdePipelineComponent {
|
||||
RdePipeline rdePipeline();
|
||||
|
||||
@Component.Builder
|
||||
interface Builder {
|
||||
@BindsInstance
|
||||
Builder options(RdePipelineOptions options);
|
||||
|
||||
RdePipelineComponent build();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.rde;
|
||||
|
||||
import google.registry.beam.common.RegistryPipelineOptions;
|
||||
import org.apache.beam.sdk.options.Description;
|
||||
|
||||
/** Custom options for running the spec11 pipeline. */
|
||||
public interface RdePipelineOptions extends RegistryPipelineOptions {
|
||||
|
||||
@Description("The Base64-encoded serialized map of TLDs to PendingDeposit.")
|
||||
String getPendings();
|
||||
|
||||
void setPendings(String value);
|
||||
|
||||
@Description("The validation mode (LENIENT|STRICT) that the RDE marshaller uses.")
|
||||
String getValidationMode();
|
||||
|
||||
void setValidationMode(String value);
|
||||
|
||||
@Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
|
||||
String getGcsBucket();
|
||||
|
||||
void setGcsBucket(String value);
|
||||
|
||||
@Description("The Base64-encoded PGP public key to encrypt the deposits.")
|
||||
String getStagingKey();
|
||||
|
||||
void setStagingKey(String value);
|
||||
}
|
||||
@@ -23,10 +23,13 @@ import dagger.Component;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.beam.common.RegistryJpaIO.Read;
|
||||
import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
|
||||
import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import google.registry.util.UtilsModule;
|
||||
@@ -96,21 +99,11 @@ public class Spec11Pipeline implements Serializable {
|
||||
}
|
||||
|
||||
void setupPipeline(Pipeline pipeline) {
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
PCollection<Subdomain> domains =
|
||||
pipeline.apply(
|
||||
"Read active domains from BigQuery",
|
||||
BigQueryIO.read(Subdomain::parseFromRecord)
|
||||
.fromQuery(
|
||||
SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
|
||||
.put("PROJECT_ID", options.getProject())
|
||||
.put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("DOMAIN_BASE_TABLE", "DomainBase")
|
||||
.build())
|
||||
.withCoder(SerializableCoder.of(Subdomain.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
options.getDatabase().equals("DATASTORE")
|
||||
? readFromBigQuery(options, pipeline)
|
||||
: readFromCloudSql(pipeline);
|
||||
|
||||
PCollection<KV<Subdomain, ThreatMatch>> threatMatches =
|
||||
domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn));
|
||||
@@ -119,6 +112,48 @@ public class Spec11Pipeline implements Serializable {
|
||||
saveToGcs(threatMatches, options);
|
||||
}
|
||||
|
||||
static PCollection<Subdomain> readFromCloudSql(Pipeline pipeline) {
|
||||
Read<Object[], Subdomain> read =
|
||||
RegistryJpaIO.read(
|
||||
"select d, r.emailAddress from Domain d join Registrar r on"
|
||||
+ " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
|
||||
+ " and d.deletionTime > now()",
|
||||
false,
|
||||
Spec11Pipeline::parseRow);
|
||||
|
||||
return pipeline.apply("Read active domains from Cloud SQL", read);
|
||||
}
|
||||
|
||||
static PCollection<Subdomain> readFromBigQuery(Spec11PipelineOptions options, Pipeline pipeline) {
|
||||
return pipeline.apply(
|
||||
"Read active domains from BigQuery",
|
||||
BigQueryIO.read(Subdomain::parseFromRecord)
|
||||
.fromQuery(
|
||||
SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
|
||||
.put("PROJECT_ID", options.getProject())
|
||||
.put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("DOMAIN_BASE_TABLE", "DomainBase")
|
||||
.build())
|
||||
.withCoder(SerializableCoder.of(Subdomain.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
}
|
||||
|
||||
private static Subdomain parseRow(Object[] row) {
|
||||
DomainBase domainBase = (DomainBase) row[0];
|
||||
String emailAddress = (String) row[1];
|
||||
if (emailAddress == null) {
|
||||
emailAddress = "";
|
||||
}
|
||||
return Subdomain.create(
|
||||
domainBase.getDomainName(),
|
||||
domainBase.getRepoId(),
|
||||
domainBase.getCurrentSponsorClientId(),
|
||||
emailAddress);
|
||||
}
|
||||
|
||||
static void saveToSql(
|
||||
PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
|
||||
String transformId = "Spec11 Threat Matches";
|
||||
|
||||
@@ -34,4 +34,9 @@ public interface Spec11PipelineOptions extends RegistryPipelineOptions {
|
||||
String getReportingBucketUrl();
|
||||
|
||||
void setReportingBucketUrl(String value);
|
||||
|
||||
@Description("The database to read data from.")
|
||||
String getDatabase();
|
||||
|
||||
void setDatabase(String value);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.config;
|
||||
|
||||
import com.google.api.gax.core.FixedCredentialsProvider;
|
||||
import com.google.cloud.tasks.v2.CloudTasksClient;
|
||||
import com.google.cloud.tasks.v2.CloudTasksSettings;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.config.CredentialModule.DefaultCredential;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import google.registry.util.GoogleCredentialsBundle;
|
||||
import google.registry.util.Retrier;
|
||||
import java.io.IOException;
|
||||
import javax.inject.Provider;
|
||||
import javax.inject.Singleton;
|
||||
|
||||
/**
|
||||
* A {@link Module} that provides {@link CloudTasksUtils}.
|
||||
*
|
||||
* <p>The class itself cannot be annotated as {@code Inject} because its requested dependencies use
|
||||
* the {@link Config} qualifier which is not available in the {@code util} package.
|
||||
*/
|
||||
@Module
|
||||
public abstract class CloudTasksUtilsModule {
|
||||
|
||||
@Singleton
|
||||
@Provides
|
||||
public static CloudTasksUtils provideCloudTasksUtils(
|
||||
@Config("projectId") String projectId,
|
||||
@Config("locationId") String locationId,
|
||||
// Use a provider so that we can use try-with-resources with the client, which implements
|
||||
// Autocloseable.
|
||||
Provider<CloudTasksClient> clientProvider,
|
||||
Retrier retrier) {
|
||||
return new CloudTasksUtils(retrier, projectId, locationId, clientProvider);
|
||||
}
|
||||
|
||||
@Provides
|
||||
public static CloudTasksClient provideCloudTasksClient(
|
||||
@DefaultCredential GoogleCredentialsBundle credentials) {
|
||||
try {
|
||||
return CloudTasksClient.create(
|
||||
CloudTasksSettings.newBuilder()
|
||||
.setCredentialsProvider(
|
||||
FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
|
||||
.build());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,7 +60,7 @@ import org.joda.time.Duration;
|
||||
* <p>This class does not represent the total configuration of the Nomulus service. It's <b>only
|
||||
* meant for settings that need to be configured <i>once</i></b>. Settings which may be subject to
|
||||
* change in the future, should instead be retrieved from Datastore. The {@link
|
||||
* google.registry.model.registry.Registry Registry} class is one such example of this.
|
||||
* google.registry.model.tld.Registry Registry} class is one such example of this.
|
||||
*
|
||||
* <p>Note: Only settings that are actually configurable belong in this file. It's not a catch-all
|
||||
* for anything widely used throughout the code base.
|
||||
@@ -105,6 +105,12 @@ public final class RegistryConfig {
|
||||
return config.appEngine.projectId;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("locationId")
|
||||
public static String provideLocationId(RegistryConfigSettings config) {
|
||||
return config.appEngine.locationId;
|
||||
}
|
||||
|
||||
/**
|
||||
* The filename of the logo to be displayed in the header of the registrar console.
|
||||
*
|
||||
@@ -416,17 +422,6 @@ public final class RegistryConfig {
|
||||
return Optional.ofNullable(config.cloudDns.servicePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns size of Google Cloud Storage client connection buffer in bytes.
|
||||
*
|
||||
* @see google.registry.gcs.GcsUtils
|
||||
*/
|
||||
@Provides
|
||||
@Config("gcsBufferSize")
|
||||
public static int provideGcsBufferSize() {
|
||||
return 1024 * 1024;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the email address of the admin account on the G Suite app used to perform
|
||||
* administrative actions.
|
||||
@@ -1278,16 +1273,34 @@ public final class RegistryConfig {
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningDays")
|
||||
public static int provideDaysToExpiration(RegistryConfigSettings config) {
|
||||
public static int provideExpirationWarningDays(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningDays;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningIntervalDays")
|
||||
public static int provideExpirationWarningIntervalDays(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningIntervalDays;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("minimumRsaKeyLength")
|
||||
public static int provideMinimumRsaKeyLength(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.minimumRsaKeyLength;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningEmailBodyText")
|
||||
public static String provideExpirationWarningEmailBodyText(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningEmailBodyText;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningEmailSubjectText")
|
||||
public static String provideExpirationWarningEmailSubjectText(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningEmailSubjectText;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("allowedEcdsaCurves")
|
||||
public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
|
||||
@@ -1387,8 +1400,8 @@ public final class RegistryConfig {
|
||||
/**
|
||||
* Returns the amount of time a domain label list should be cached in memory before expiring.
|
||||
*
|
||||
* @see google.registry.model.registry.label.ReservedList
|
||||
* @see google.registry.model.registry.label.PremiumList
|
||||
* @see google.registry.model.tld.label.ReservedList
|
||||
* @see google.registry.model.tld.label.PremiumList
|
||||
*/
|
||||
public static Duration getDomainLabelListCacheDuration() {
|
||||
return Duration.standardSeconds(CONFIG_SETTINGS.get().caching.domainLabelCachingSeconds);
|
||||
@@ -1487,21 +1500,6 @@ public final class RegistryConfig {
|
||||
return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether to replicate cloud SQL transactions to datastore.
|
||||
*
|
||||
* <p>If true, all cloud SQL transactions will be persisted as TransactionEntity objects in the
|
||||
* Transaction table and replayed against datastore in a cron job.
|
||||
*/
|
||||
public static boolean getCloudSqlReplicateTransactions() {
|
||||
return CONFIG_SETTINGS.get().cloudSql.replicateTransactions;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void overrideCloudSqlReplicateTransactions(boolean replicateTransactions) {
|
||||
CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
|
||||
}
|
||||
|
||||
/** Returns the roid suffix to be used for the roids of all contacts and hosts. */
|
||||
public static String getContactAndHostRoidSuffix() {
|
||||
return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;
|
||||
|
||||
@@ -45,6 +45,7 @@ public class RegistryConfigSettings {
|
||||
/** Configuration options that apply to the entire App Engine project. */
|
||||
public static class AppEngine {
|
||||
public String projectId;
|
||||
public String locationId;
|
||||
public boolean isLocal;
|
||||
public String defaultServiceUrl;
|
||||
public String backendServiceUrl;
|
||||
@@ -126,7 +127,6 @@ public class RegistryConfigSettings {
|
||||
// TODO(05012021): remove username field after it is removed from all yaml files.
|
||||
public String username;
|
||||
public String instanceConnectionName;
|
||||
public boolean replicateTransactions;
|
||||
}
|
||||
|
||||
/** Configuration for Apache Beam (Cloud Dataflow). */
|
||||
@@ -228,7 +228,10 @@ public class RegistryConfigSettings {
|
||||
public static class SslCertificateValidation {
|
||||
public Map<String, Integer> maxValidityDaysSchedule;
|
||||
public int expirationWarningDays;
|
||||
public int expirationWarningIntervalDays;
|
||||
public int minimumRsaKeyLength;
|
||||
public Set<String> allowedEcdsaCurves;
|
||||
public String expirationWarningEmailBodyText;
|
||||
public String expirationWarningEmailSubjectText;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,10 @@
|
||||
appEngine:
|
||||
# Globally unique App Engine project ID
|
||||
projectId: registry-project-id
|
||||
# Location of the App engine project, note that us-central1 and europe-west1 are special in that
|
||||
# they are used without the trailing number in App Engine commands and Google Cloud Console.
|
||||
# See: https://cloud.google.com/appengine/docs/locations
|
||||
locationId: registry-location-id
|
||||
|
||||
# whether to use local/test credentials when connecting to the servers
|
||||
isLocal: true
|
||||
@@ -227,9 +231,6 @@ cloudSql:
|
||||
jdbcUrl: jdbc:postgresql://localhost
|
||||
# This name is used by Cloud SQL when connecting to the database.
|
||||
instanceConnectionName: project-id:region:instance-id
|
||||
# Set this to true to replicate cloud SQL transactions to datastore in the
|
||||
# background.
|
||||
replicateTransactions: false
|
||||
|
||||
cloudDns:
|
||||
# Set both properties to null in Production.
|
||||
@@ -452,6 +453,56 @@ sslCertificateValidation:
|
||||
# The number of days before a certificate expires that indicates the
|
||||
# certificate is nearing expiration and warnings should be sent.
|
||||
expirationWarningDays: 30
|
||||
# The minimum number of days between two successive expiring notification emails.
|
||||
expirationWarningIntervalDays: 15
|
||||
# Text for expiring certificate notification email subject.
|
||||
expirationWarningEmailSubjectText: "[Important] Expiring SSL certificate for Google Registry EPP connection"
|
||||
# Text for expiring certificate notification email body that accepts 3 parameters:
|
||||
# registrar name, certificate type, and expiration date, respectively.
|
||||
expirationWarningEmailBodyText: >
|
||||
Dear %1$s,
|
||||
|
||||
We would like to inform you that your %2$s SSL certificate will expire at
|
||||
%3$s. Please take note that using expired certificates will prevent
|
||||
successful Registry login.
|
||||
|
||||
Kindly update your production account certificate within the support
|
||||
console using the following steps:
|
||||
|
||||
1. Navigate to support.registry.google and login using your
|
||||
%4$s@registry.google credentials.
|
||||
* If this is your first time logging in, you will be prompted to
|
||||
reset your password, so please keep your new password safe.
|
||||
* If you are already logged in with some other Google account(s) but
|
||||
not your %4$s@registry.google account, you need to click on
|
||||
“Add Account” and login using your %4$s@registry.google credentials.
|
||||
2. Select “Settings > Security” from the left navigation bar.
|
||||
3. Click “Edit” on the top left corner.
|
||||
4. Enter your full certificate string
|
||||
(including lines -----BEGIN CERTIFICATE----- and
|
||||
-----END CERTIFICATE-----) in the box.
|
||||
5. Click “Save”. If there are validation issues with the form, you will
|
||||
be prompted to fix them and click “Save” again.
|
||||
|
||||
A failover SSL certificate can also be added in order to prevent connection
|
||||
issues once your main certificate expires. Connecting with either of the
|
||||
certificates will work with our production EPP server.
|
||||
|
||||
Further information about our EPP connection requirements can be found in
|
||||
section 9.2 in the updated Technical Guide in your Google Drive folder.
|
||||
|
||||
Note that account certificate changes take a few minutes to become
|
||||
effective and that the existing connections will remain unaffected by
|
||||
the change.
|
||||
|
||||
If you also would like to update your OT&E account certificate, please send
|
||||
an email from your primary or technical contact to
|
||||
registry-support@google.com and include the full certificate string
|
||||
(including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----).
|
||||
|
||||
Regards,
|
||||
Google Registry
|
||||
|
||||
# The minimum number of bits an RSA key must contain.
|
||||
minimumRsaKeyLength: 2048
|
||||
# The ECDSA curves that are allowed for public keys.
|
||||
|
||||
@@ -14,14 +14,10 @@
|
||||
|
||||
package google.registry.cron;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Predicates.in;
|
||||
import static com.google.common.base.Predicates.not;
|
||||
import static com.google.common.base.Strings.nullToEmpty;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static com.google.common.collect.Iterables.getFirst;
|
||||
import static com.google.common.collect.Multimaps.filterKeys;
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static google.registry.cron.CronModule.ENDPOINT_PARAM;
|
||||
@@ -31,26 +27,29 @@ import static google.registry.cron.CronModule.FOR_EACH_TEST_TLD_PARAM;
|
||||
import static google.registry.cron.CronModule.JITTER_SECONDS_PARAM;
|
||||
import static google.registry.cron.CronModule.QUEUE_PARAM;
|
||||
import static google.registry.cron.CronModule.RUN_IN_EMPTY_PARAM;
|
||||
import static google.registry.model.registry.Registries.getTldsOfType;
|
||||
import static google.registry.model.registry.Registry.TldType.REAL;
|
||||
import static google.registry.model.registry.Registry.TldType.TEST;
|
||||
import static google.registry.model.tld.Registries.getTldsOfType;
|
||||
import static google.registry.model.tld.Registry.TldType.REAL;
|
||||
import static google.registry.model.tld.Registry.TldType.TEST;
|
||||
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.cloud.tasks.v2.Task;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.ImmutableListMultimap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Multimap;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.protobuf.Timestamp;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Service;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.ParameterMap;
|
||||
import google.registry.request.RequestParameters;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.TaskQueueUtils;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
import java.util.Random;
|
||||
import java.util.stream.Stream;
|
||||
@@ -105,7 +104,8 @@ public final class TldFanoutAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject TaskQueueUtils taskQueueUtils;
|
||||
@Inject Clock clock;
|
||||
@Inject CloudTasksUtils cloudTasksUtils;
|
||||
@Inject Response response;
|
||||
@Inject @Parameter(ENDPOINT_PARAM) String endpoint;
|
||||
@Inject @Parameter(QUEUE_PARAM) String queue;
|
||||
@@ -115,7 +115,9 @@ public final class TldFanoutAction implements Runnable {
|
||||
@Inject @Parameter(EXCLUDE_PARAM) ImmutableSet<String> excludes;
|
||||
@Inject @Parameter(JITTER_SECONDS_PARAM) Optional<Integer> jitterSeconds;
|
||||
@Inject @ParameterMap ImmutableListMultimap<String, String> params;
|
||||
@Inject TldFanoutAction() {}
|
||||
|
||||
@Inject
|
||||
TldFanoutAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
@@ -126,8 +128,7 @@ public final class TldFanoutAction implements Runnable {
|
||||
runInEmpty || forEachTestTld || forEachRealTld,
|
||||
"At least one of runInEmpty, forEachTestTld, forEachRealTld must be given");
|
||||
checkArgument(
|
||||
!(runInEmpty && !excludes.isEmpty()),
|
||||
"Can't specify 'exclude' with 'runInEmpty'");
|
||||
!(runInEmpty && !excludes.isEmpty()), "Can't specify 'exclude' with 'runInEmpty'");
|
||||
ImmutableSet<String> tlds =
|
||||
runInEmpty
|
||||
? ImmutableSet.of("")
|
||||
@@ -137,7 +138,6 @@ public final class TldFanoutAction implements Runnable {
|
||||
.filter(not(in(excludes)))
|
||||
.collect(toImmutableSet());
|
||||
Multimap<String, String> flowThruParams = filterKeys(params, not(in(CONTROL_PARAMS)));
|
||||
Queue taskQueue = getQueue(queue);
|
||||
StringBuilder outputPayload =
|
||||
new StringBuilder(
|
||||
String.format("OK: Launched the following %d tasks in queue %s\n", tlds.size(), queue));
|
||||
@@ -146,33 +146,41 @@ public final class TldFanoutAction implements Runnable {
|
||||
logger.atWarning().log("No TLDs to fan-out!");
|
||||
}
|
||||
for (String tld : tlds) {
|
||||
TaskOptions taskOptions = createTaskOptions(tld, flowThruParams);
|
||||
TaskHandle taskHandle = taskQueueUtils.enqueue(taskQueue, taskOptions);
|
||||
Task task = createTask(tld, flowThruParams);
|
||||
Task createdTask = cloudTasksUtils.enqueue(queue, task);
|
||||
outputPayload.append(
|
||||
String.format(
|
||||
"- Task: '%s', tld: '%s', endpoint: '%s'\n",
|
||||
taskHandle.getName(), tld, taskOptions.getUrl()));
|
||||
createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri()));
|
||||
logger.atInfo().log(
|
||||
"Task: '%s', tld: '%s', endpoint: '%s'", taskHandle.getName(), tld, taskOptions.getUrl());
|
||||
"Task: '%s', tld: '%s', endpoint: '%s'",
|
||||
createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri());
|
||||
}
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
response.setPayload(outputPayload.toString());
|
||||
}
|
||||
|
||||
private TaskOptions createTaskOptions(String tld, Multimap<String, String> params) {
|
||||
TaskOptions options =
|
||||
withUrl(endpoint)
|
||||
.countdownMillis(
|
||||
jitterSeconds
|
||||
.map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
|
||||
.orElse(0));
|
||||
private Task createTask(String tld, Multimap<String, String> params) {
|
||||
if (!tld.isEmpty()) {
|
||||
options.param(RequestParameters.PARAM_TLD, tld);
|
||||
params = ArrayListMultimap.create(params);
|
||||
params.put(RequestParameters.PARAM_TLD, tld);
|
||||
}
|
||||
for (String param : params.keySet()) {
|
||||
// TaskOptions.param() does not accept null values.
|
||||
options.param(param, nullToEmpty(getFirst(params.get(param), null)));
|
||||
}
|
||||
return options;
|
||||
Instant scheduleTime =
|
||||
Instant.ofEpochMilli(
|
||||
clock
|
||||
.nowUtc()
|
||||
.plusMillis(
|
||||
jitterSeconds
|
||||
.map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
|
||||
.orElse(0))
|
||||
.getMillis());
|
||||
return Task.newBuilder(
|
||||
CloudTasksUtils.createPostTask(endpoint, Service.BACKEND.toString(), params))
|
||||
.setScheduleTime(
|
||||
Timestamp.newBuilder()
|
||||
.setSeconds(scheduleTime.getEpochSecond())
|
||||
.setNanos(scheduleTime.getNano())
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ import static google.registry.dns.DnsConstants.DNS_PULL_QUEUE_NAME;
|
||||
import static google.registry.dns.DnsConstants.DNS_TARGET_CREATE_TIME_PARAM;
|
||||
import static google.registry.dns.DnsConstants.DNS_TARGET_NAME_PARAM;
|
||||
import static google.registry.dns.DnsConstants.DNS_TARGET_TYPE_PARAM;
|
||||
import static google.registry.model.registry.Registries.assertTldExists;
|
||||
import static google.registry.model.tld.Registries.assertTldExists;
|
||||
import static google.registry.request.RequestParameters.PARAM_TLD;
|
||||
import static google.registry.util.DomainNameUtils.getTldFromDomainName;
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
@@ -38,7 +38,7 @@ import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.InternetDomainName;
|
||||
import com.google.common.util.concurrent.RateLimiter;
|
||||
import google.registry.dns.DnsConstants.TargetType;
|
||||
import google.registry.model.registry.Registries;
|
||||
import google.registry.model.tld.Registries;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import google.registry.util.SystemClock;
|
||||
|
||||
@@ -19,7 +19,7 @@ import static com.google.common.base.Preconditions.checkState;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.dns.writer.DnsWriter;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.tld.Registry;
|
||||
import java.util.Map;
|
||||
import javax.inject.Inject;
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ import google.registry.dns.DnsMetrics.ActionStatus;
|
||||
import google.registry.dns.DnsMetrics.CommitStatus;
|
||||
import google.registry.dns.DnsMetrics.PublishStatus;
|
||||
import google.registry.dns.writer.DnsWriter;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.HttpException.ServiceUnavailableException;
|
||||
import google.registry.request.Parameter;
|
||||
|
||||
@@ -47,8 +47,8 @@ import com.google.common.hash.HashFunction;
|
||||
import com.google.common.hash.Hashing;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.dns.DnsConstants.TargetType;
|
||||
import google.registry.model.registry.Registries;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.tld.Registries;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.auth.Auth;
|
||||
|
||||
@@ -39,7 +39,7 @@ import google.registry.dns.writer.DnsWriterZone;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.secdns.DelegationSignerData;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.registry.Registries;
|
||||
import google.registry.model.tld.Registries;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.Concurrent;
|
||||
import google.registry.util.Retrier;
|
||||
@@ -284,9 +284,10 @@ public class CloudDnsWriter extends BaseDnsWriter {
|
||||
.filter(hostName -> hostName.endsWith("." + domainName) && !hostName.equals(domainName));
|
||||
}
|
||||
|
||||
/** Mutate the zone with the provided {@code desiredRecords}. */
|
||||
/** Mutate the zone with the provided map of hostnames to desired DNS records. */
|
||||
@VisibleForTesting
|
||||
void mutateZone(ImmutableMap<String, ImmutableSet<ResourceRecordSet>> desiredRecords) {
|
||||
logger.atInfo().log("Updating DNS records for hostname(s) %s.", desiredRecords.keySet());
|
||||
// Fetch all existing records for names that this writer is trying to modify
|
||||
ImmutableSet.Builder<ResourceRecordSet> flattenedExistingRecords = new ImmutableSet.Builder<>();
|
||||
|
||||
@@ -313,7 +314,12 @@ public class CloudDnsWriter extends BaseDnsWriter {
|
||||
desiredRecords.values().forEach(flattenedDesiredRecords::addAll);
|
||||
|
||||
// Delete all existing records and add back the desired records
|
||||
updateResourceRecords(flattenedDesiredRecords.build(), flattenedExistingRecords.build());
|
||||
try {
|
||||
updateResourceRecords(flattenedDesiredRecords.build(), flattenedExistingRecords.build());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(
|
||||
"Error updating DNS records for hostname(s) " + desiredRecords.keySet(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -363,10 +369,12 @@ public class CloudDnsWriter extends BaseDnsWriter {
|
||||
*
|
||||
* @throws ZoneStateException if the operation could not be completely successfully because the
|
||||
* records to delete do not exist, already exist or have been modified with different
|
||||
* attributes since being queried.
|
||||
* attributes since being queried. These errors will be retried.
|
||||
* @throws IOException on non-retryable API errors, e.g. invalid request.
|
||||
*/
|
||||
private void updateResourceRecords(
|
||||
ImmutableSet<ResourceRecordSet> additions, ImmutableSet<ResourceRecordSet> deletions) {
|
||||
ImmutableSet<ResourceRecordSet> additions, ImmutableSet<ResourceRecordSet> deletions)
|
||||
throws IOException {
|
||||
// Find records that are both in additions and deletions, so we can remove them from both before
|
||||
// requesting the change. This is mostly for optimization reasons - not doing so doesn't affect
|
||||
// the result.
|
||||
@@ -392,17 +400,15 @@ public class CloudDnsWriter extends BaseDnsWriter {
|
||||
GoogleJsonError err = e.getDetails();
|
||||
// We did something really wrong here, just give up and re-throw
|
||||
if (err == null || err.getErrors().size() > 1) {
|
||||
throw new RuntimeException(e);
|
||||
throw e;
|
||||
}
|
||||
String errorReason = err.getErrors().get(0).getReason();
|
||||
|
||||
if (RETRYABLE_EXCEPTION_REASONS.contains(errorReason)) {
|
||||
throw new ZoneStateException(errorReason);
|
||||
} else {
|
||||
throw new RuntimeException(e);
|
||||
throw e;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ import google.registry.dns.writer.DnsWriterZone;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.secdns.DelegationSignerData;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.registry.Registries;
|
||||
import google.registry.model.tld.Registries;
|
||||
import google.registry.util.Clock;
|
||||
import java.io.IOException;
|
||||
import java.net.Inet4Address;
|
||||
|
||||
@@ -199,6 +199,15 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
|
||||
<description>
|
||||
|
||||
@@ -237,6 +237,12 @@
|
||||
<url-pattern>/_dr/task/killCommitLogs</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Replays Datastore commit logs to SQL. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/replayCommitLogsToSql</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- MapReduce servlet. -->
|
||||
<servlet>
|
||||
<servlet-name>mapreduce</servlet-name>
|
||||
@@ -349,6 +355,12 @@
|
||||
<url-pattern>/_dr/task/deleteExpiredDomains</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Background action to send notification emails to registrars with expiring certificate. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/sendExpiringCertificateNotificationEmail</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to import contacts from escrow file -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
|
||||
@@ -102,6 +102,7 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<!-- Disabled for sql-only tests.
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
|
||||
<description>
|
||||
@@ -110,6 +111,7 @@
|
||||
<schedule>every 3 minutes synchronized</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
-->
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/task/deleteContactsAndHosts]]></url>
|
||||
@@ -133,6 +135,7 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<!-- Disabled for sql-only tests
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
|
||||
<description>
|
||||
@@ -140,16 +143,14 @@
|
||||
It also enqueues a new task to wait on the completion of that job and then load the resulting
|
||||
snapshot into bigquery.
|
||||
</description>
|
||||
<!--
|
||||
<!- -
|
||||
Keep google.registry.export.CheckBackupAction.MAXIMUM_BACKUP_RUNNING_TIME less than
|
||||
this interval. -->
|
||||
this interval. - ->
|
||||
<schedule>every day 06:00</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
-->
|
||||
|
||||
<!--
|
||||
Removed for the duration of load testing
|
||||
TODO(b/71607184): Restore after loadtesting is done
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
|
||||
<description>
|
||||
@@ -159,7 +160,6 @@
|
||||
<timezone>UTC</timezone>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
-->
|
||||
|
||||
<!-- TODO: Add borgmon job to check that these files are created and updated successfully. -->
|
||||
<cron>
|
||||
@@ -200,4 +200,35 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<!-- Disabled for sql-only tests.
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
-->
|
||||
|
||||
<!--
|
||||
The next two wipeout jobs are required when crash has production data.
|
||||
-->
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/task/wipeOutCloudSql]]></url>
|
||||
<description>
|
||||
This job runs an action that deletes all data in Cloud SQL.
|
||||
</description>
|
||||
<schedule>every saturday 03:07</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/task/wipeOutDatastore]]></url>
|
||||
<description>
|
||||
This job runs an action that deletes all data in Cloud Datastore.
|
||||
</description>
|
||||
<schedule>every saturday 03:07</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
</cronentries>
|
||||
|
||||
@@ -330,4 +330,13 @@
|
||||
<schedule>every day 15:00</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
</cronentries>
|
||||
|
||||
@@ -1,6 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<cronentries>
|
||||
|
||||
<cron>
|
||||
<url>/_dr/task/rdeStaging</url>
|
||||
<description>
|
||||
This job generates a full RDE escrow deposit as a single gigantic XML document
|
||||
and streams it to cloud storage. When this job has finished successfully, it'll
|
||||
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
|
||||
</description>
|
||||
<schedule>every day 00:07</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
|
||||
<description>
|
||||
@@ -100,4 +111,21 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
|
||||
<description>
|
||||
This job checkpoints the commit log buckets and exports the diff since last checkpoint to GCS.
|
||||
</description>
|
||||
<schedule>every 3 minutes synchronized</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
</cronentries>
|
||||
|
||||
@@ -168,6 +168,15 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/task/sendExpiringCertificateNotificationEmail]]></url>
|
||||
<description>
|
||||
This job runs an action that sends emails to partners if their certificates are expiring soon.
|
||||
</description>
|
||||
<schedule>every day 04:30</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
|
||||
<description>
|
||||
@@ -229,4 +238,12 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
</cronentries>
|
||||
|
||||
@@ -14,20 +14,19 @@
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.base.Verify.verifyNotNull;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
import static google.registry.model.registry.Registries.getTldsOfType;
|
||||
import static google.registry.model.tld.Registries.getTldsOfType;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Suppliers;
|
||||
@@ -39,18 +38,20 @@ import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.model.tld.Registry.TldType;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.List;
|
||||
import java.util.function.Supplier;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -70,26 +71,112 @@ public class ExportDomainListsAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final int MAX_NUM_REDUCE_SHARDS = 100;
|
||||
public static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject DriveConnection driveConnection;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("domainListsGcsBucket") String gcsBucket;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject ExportDomainListsAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
|
||||
logger.atInfo().log("Exporting domain lists for tlds %s", realTlds);
|
||||
mrRunner
|
||||
.setJobName("Export domain lists")
|
||||
.setModuleName("backend")
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(DateTime.now(UTC), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Export domain lists")
|
||||
.setModuleName("backend")
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(clock.nowUtc(), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsUtils),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
realTlds.forEach(
|
||||
tld -> {
|
||||
List<String> domains =
|
||||
tm().transact(
|
||||
() ->
|
||||
// Note that if we had "creationTime <= :now" in the condition (not
|
||||
// necessary as there is no pending creation, the order of deletionTime
|
||||
// and creationTime in the query would have been significant and it
|
||||
// should come after deletionTime. When Hibernate substitutes "now" it
|
||||
// will first validate that the **first** field that is to be compared
|
||||
// with it (deletionTime) is assignable from the substituted Java object
|
||||
// (click.nowUtc()). Since creationTime is a CreateAutoTimestamp, if it
|
||||
// comes first, we will need to substitute "now" with
|
||||
// CreateAutoTimestamp.create(clock.nowUtc()). This might look a bit
|
||||
// strange as the Java object type is clearly incompatible between the
|
||||
// two fields deletionTime (DateTime) and creationTime, yet they are
|
||||
// compared with the same "now". It is actually OK because in the end
|
||||
// Hibernate converts everything to SQL types (and Java field names to
|
||||
// SQL column names) to run the query. Both CreateAutoTimestamp and
|
||||
// DateTime are persisted as timestamp_z in SQL. It is only the
|
||||
// validation that compares the Java types, and only with the first
|
||||
// field that compares with the substituted value.
|
||||
jpaTm()
|
||||
.query(
|
||||
"SELECT fullyQualifiedDomainName FROM Domain "
|
||||
+ "WHERE tld = :tld "
|
||||
+ "AND deletionTime > :now "
|
||||
+ "ORDER by fullyQualifiedDomainName ASC",
|
||||
String.class)
|
||||
.setParameter("tld", tld)
|
||||
.setParameter("now", clock.nowUtc())
|
||||
.getResultList());
|
||||
String domainsList = Joiner.on("\n").join(domains);
|
||||
logger.atInfo().log(
|
||||
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsUtils);
|
||||
exportToDrive(tld, domainsList, driveConnection);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
protected static boolean exportToDrive(
|
||||
String tld, String domains, DriveConnection driveConnection) {
|
||||
verifyNotNull(driveConnection, "Expecting non-null driveConnection");
|
||||
try {
|
||||
Registry registry = Registry.get(tld);
|
||||
if (registry.getDriveFolderId() == null) {
|
||||
logger.atInfo().log(
|
||||
"Skipping registered domains export for TLD %s because Drive folder isn't specified",
|
||||
tld);
|
||||
} else {
|
||||
String resultMsg =
|
||||
driveConnection.createOrUpdateFile(
|
||||
REGISTERED_DOMAINS_FILENAME,
|
||||
MediaType.PLAIN_TEXT_UTF_8,
|
||||
registry.getDriveFolderId(),
|
||||
domains.getBytes(UTF_8));
|
||||
logger.atInfo().log(
|
||||
"Exporting registered domains succeeded for TLD %s, response was: %s", tld, resultMsg);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to Drive, skipping...", tld);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
protected static boolean exportToGcs(
|
||||
String tld, String domains, String gcsBucket, GcsUtils gcsUtils) {
|
||||
BlobId blobId = BlobId.of(gcsBucket, tld + ".txt");
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to GCS, skipping...", tld);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static class ExportDomainListsMapper extends Mapper<DomainBase, String, String> {
|
||||
@@ -122,11 +209,8 @@ public class ExportDomainListsAction implements Runnable {
|
||||
private static Supplier<DriveConnection> driveConnectionSupplier =
|
||||
Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection());
|
||||
|
||||
static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
|
||||
static final MediaType EXPORT_MIME_TYPE = MediaType.PLAIN_TEXT_UTF_8;
|
||||
|
||||
private final String gcsBucket;
|
||||
private final int gcsBufferSize;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
/**
|
||||
* Non-serializable {@link DriveConnection} that will be created when an instance of {@link
|
||||
@@ -136,9 +220,9 @@ public class ExportDomainListsAction implements Runnable {
|
||||
*/
|
||||
private transient DriveConnection driveConnection;
|
||||
|
||||
public ExportDomainListsReducer(String gcsBucket, int gcsBufferSize) {
|
||||
public ExportDomainListsReducer(String gcsBucket, GcsUtils gcsUtils) {
|
||||
this.gcsBucket = gcsBucket;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@@ -147,53 +231,21 @@ public class ExportDomainListsAction implements Runnable {
|
||||
driveConnection = driveConnectionSupplier.get();
|
||||
}
|
||||
|
||||
private void exportToDrive(String tld, String domains) {
|
||||
verifyNotNull(driveConnection, "expecting non-null driveConnection");
|
||||
try {
|
||||
Registry registry = Registry.get(tld);
|
||||
if (registry.getDriveFolderId() == null) {
|
||||
logger.atInfo().log(
|
||||
"Skipping registered domains export for TLD %s because Drive folder isn't specified",
|
||||
tld);
|
||||
} else {
|
||||
String resultMsg =
|
||||
driveConnection.createOrUpdateFile(
|
||||
REGISTERED_DOMAINS_FILENAME,
|
||||
EXPORT_MIME_TYPE,
|
||||
registry.getDriveFolderId(),
|
||||
domains.getBytes(UTF_8));
|
||||
logger.atInfo().log(
|
||||
"Exporting registered domains succeeded for TLD %s, response was: %s",
|
||||
tld, resultMsg);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to Drive", tld);
|
||||
}
|
||||
getContext().incrementCounter("domain lists written out to Drive");
|
||||
}
|
||||
|
||||
private void exportToGcs(String tld, String domains) {
|
||||
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (IOException e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to GCS.", tld);
|
||||
}
|
||||
getContext().incrementCounter("domain lists written out to GCS");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reduce(String tld, ReducerInput<String> fqdns) {
|
||||
ImmutableList<String> domains = ImmutableList.sortedCopyOf(() -> fqdns);
|
||||
String domainsList = Joiner.on('\n').join(domains);
|
||||
logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList);
|
||||
exportToDrive(tld, domainsList);
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsUtils)) {
|
||||
getContext().incrementCounter("domain lists successful written out to GCS");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to GCS");
|
||||
}
|
||||
if (exportToDrive(tld, domainsList, driveConnection)) {
|
||||
getContext().incrementCounter("domain lists successfully written out to Drive");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to Drive");
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
|
||||
@@ -30,14 +30,14 @@ import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.model.tld.label.PremiumList.PremiumEntry;
|
||||
import google.registry.model.tld.label.PremiumListDao;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.RequestParameters;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.schema.tld.PremiumListDao;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
@@ -113,7 +113,7 @@ public class ExportPremiumTermsAction implements Runnable {
|
||||
"Skipping premium terms export for TLD %s because Drive folder isn't specified", tld);
|
||||
return Optional.of("Skipping export because no Drive folder is associated with this TLD");
|
||||
}
|
||||
if (!registry.getPremiumList().isPresent()) {
|
||||
if (!registry.getPremiumListName().isPresent()) {
|
||||
logger.atInfo().log("No premium terms to export for TLD %s", tld);
|
||||
return Optional.of("No premium lists configured");
|
||||
}
|
||||
@@ -137,14 +137,14 @@ public class ExportPremiumTermsAction implements Runnable {
|
||||
}
|
||||
|
||||
private String getFormattedPremiumTerms(Registry registry) {
|
||||
checkState(registry.getPremiumList().isPresent(), "%s does not have a premium list", tld);
|
||||
String premiumListName = registry.getPremiumList().get().getName();
|
||||
checkState(registry.getPremiumListName().isPresent(), "%s does not have a premium list", tld);
|
||||
String premiumListName = registry.getPremiumListName().get();
|
||||
checkState(
|
||||
PremiumListDao.getLatestRevision(premiumListName).isPresent(),
|
||||
"Could not load premium list for " + tld);
|
||||
SortedSet<String> premiumTerms =
|
||||
Streams.stream(PremiumListDao.loadAllPremiumListEntries(premiumListName))
|
||||
.map(PremiumListEntry::toString)
|
||||
Streams.stream(PremiumListDao.loadAllPremiumEntries(premiumListName))
|
||||
.map(PremiumEntry::toString)
|
||||
.collect(ImmutableSortedSet.toImmutableSortedSet(String::compareTo));
|
||||
|
||||
return Joiner.on("\n")
|
||||
|
||||
@@ -23,7 +23,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.RequestParameters;
|
||||
@@ -63,7 +63,7 @@ public class ExportReservedTermsAction implements Runnable {
|
||||
try {
|
||||
Registry registry = Registry.get(tld);
|
||||
String resultMsg;
|
||||
if (registry.getReservedLists().isEmpty() && isNullOrEmpty(registry.getDriveFolderId())) {
|
||||
if (registry.getReservedListNames().isEmpty() && isNullOrEmpty(registry.getDriveFolderId())) {
|
||||
resultMsg = "No reserved lists configured";
|
||||
logger.atInfo().log("No reserved terms to export for TLD %s", tld);
|
||||
} else if (registry.getDriveFolderId() == null) {
|
||||
|
||||
@@ -15,11 +15,11 @@
|
||||
package google.registry.export;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.registry.label.ReservedList.ReservedListEntry;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.model.tld.label.ReservedList;
|
||||
import google.registry.model.tld.label.ReservedList.ReservedListEntry;
|
||||
import google.registry.model.tld.label.ReservedListDao;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import javax.inject.Inject;
|
||||
@@ -39,11 +39,16 @@ public final class ExportUtils {
|
||||
public String exportReservedTerms(Registry registry) {
|
||||
StringBuilder termsBuilder = new StringBuilder(reservedTermsExportDisclaimer).append("\n");
|
||||
Set<String> reservedTerms = new TreeSet<>();
|
||||
for (Key<ReservedList> key : registry.getReservedLists()) {
|
||||
ReservedList reservedList = ReservedList.load(key).get();
|
||||
for (String reservedListName : registry.getReservedListNames()) {
|
||||
ReservedList reservedList =
|
||||
ReservedListDao.getLatestRevision(reservedListName)
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalStateException(
|
||||
String.format("Reserved list %s does not exist", reservedListName)));
|
||||
if (reservedList.getShouldPublish()) {
|
||||
for (ReservedListEntry entry : reservedList.getReservedListEntries().values()) {
|
||||
reservedTerms.add(entry.getLabel());
|
||||
reservedTerms.add(entry.getDomainLabel());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ import static com.google.common.net.HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN;
|
||||
import static google.registry.flows.domain.DomainFlowUtils.validateDomainName;
|
||||
import static google.registry.flows.domain.DomainFlowUtils.validateDomainNameWithIdnTables;
|
||||
import static google.registry.flows.domain.DomainFlowUtils.verifyNotInPredelegation;
|
||||
import static google.registry.model.registry.label.ReservationType.getTypeOfHighestSeverity;
|
||||
import static google.registry.model.registry.label.ReservedList.getReservationTypes;
|
||||
import static google.registry.model.tld.label.ReservationType.getTypeOfHighestSeverity;
|
||||
import static google.registry.model.tld.label.ReservedList.getReservationTypes;
|
||||
import static google.registry.monitoring.whitebox.CheckApiMetric.Availability.AVAILABLE;
|
||||
import static google.registry.monitoring.whitebox.CheckApiMetric.Availability.REGISTERED;
|
||||
import static google.registry.monitoring.whitebox.CheckApiMetric.Availability.RESERVED;
|
||||
@@ -46,8 +46,8 @@ import google.registry.flows.domain.DomainFlowUtils.BadCommandForRegistryPhaseEx
|
||||
import google.registry.flows.domain.DomainFlowUtils.InvalidIdnDomainLabelException;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.index.ForeignKeyIndex;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.label.ReservationType;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.model.tld.label.ReservationType;
|
||||
import google.registry.monitoring.whitebox.CheckApiMetric;
|
||||
import google.registry.monitoring.whitebox.CheckApiMetric.Availability;
|
||||
import google.registry.request.Action;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user