mirror of
https://github.com/google/nomulus
synced 2026-05-25 09:10:51 +00:00
Compare commits
59 Commits
nomulus-20
...
nomulus-20
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2754a0eff | ||
|
|
276bbc09c2 | ||
|
|
fd461a78e7 | ||
|
|
0374ad60d8 | ||
|
|
fcc027e0c8 | ||
|
|
c3a4887845 | ||
|
|
a0b6437f4c | ||
|
|
a7210a26b4 | ||
|
|
c7096a1b71 | ||
|
|
30634ff404 | ||
|
|
4f71d780ab | ||
|
|
14ad56a392 | ||
|
|
a1b56b0521 | ||
|
|
3f41f7f444 | ||
|
|
4f6bcea63f | ||
|
|
bd0ef626a1 | ||
|
|
68304133c4 | ||
|
|
16392c3808 | ||
|
|
5f479488fa | ||
|
|
886a970ed6 | ||
|
|
d7f7568761 | ||
|
|
2017930a8f | ||
|
|
ed07fc8181 | ||
|
|
aa2898ebfc | ||
|
|
586189d7ee | ||
|
|
275f364dcb | ||
|
|
66867e4397 | ||
|
|
3fa56dec45 | ||
|
|
92f5f8989b | ||
|
|
810adf0158 | ||
|
|
f6004181f8 | ||
|
|
296440b277 | ||
|
|
50f80744d8 | ||
|
|
826320c7fd | ||
|
|
8099789012 | ||
|
|
20a0e4ce3f | ||
|
|
2f2e9dd49f | ||
|
|
5e28694053 | ||
|
|
642405375b | ||
|
|
02eb7cfcc3 | ||
|
|
f7dca7fa96 | ||
|
|
a7e8ae5a2c | ||
|
|
dc7f21ca68 | ||
|
|
e96873f2d0 | ||
|
|
b5f05405a0 | ||
|
|
f702f2670b | ||
|
|
21aeedae11 | ||
|
|
c1f0c29134 | ||
|
|
16641e05a1 | ||
|
|
bf1c34cc3b | ||
|
|
93dc812ea2 | ||
|
|
e09138645f | ||
|
|
238deb25ec | ||
|
|
6ce2926c6d | ||
|
|
27f431b9cf | ||
|
|
2bb0e7305d | ||
|
|
10757863ce | ||
|
|
02079010c6 | ||
|
|
4246e7e4e0 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -4,6 +4,7 @@
|
||||
######################################################################
|
||||
# Java Ignores
|
||||
|
||||
gjf.out
|
||||
*.class
|
||||
|
||||
# Mobile Tools for Java (J2ME)
|
||||
|
||||
@@ -318,7 +318,7 @@ subprojects {
|
||||
// expose to users.
|
||||
if (project.name != 'docs') {
|
||||
javadocSource << project.sourceSets.main.allJava
|
||||
javadocClasspath << project.sourceSets.main.compileClasspath
|
||||
javadocClasspath << project.sourceSets.main.runtimeClasspath
|
||||
javadocClasspath << "${buildDir}/generated/sources/annotationProcessor/java/main"
|
||||
javadocDependentTasks << project.tasks.compileJava
|
||||
}
|
||||
@@ -457,6 +457,8 @@ task javaIncrementalFormatApply {
|
||||
task javadoc(type: Javadoc) {
|
||||
source javadocSource
|
||||
classpath = files(javadocClasspath)
|
||||
// Exclude the misbehaving generated-by-Soy Java files
|
||||
exclude "**/*SoyInfo.java"
|
||||
destinationDir = file("${buildDir}/docs/javadoc")
|
||||
options.encoding = "UTF-8"
|
||||
// In a lot of places we don't write @return so suppress warnings about that.
|
||||
|
||||
@@ -72,6 +72,7 @@ dependencies {
|
||||
compile deps['com.google.auth:google-auth-library-credentials']
|
||||
compile deps['com.google.auth:google-auth-library-oauth2-http']
|
||||
compile deps['com.google.auto.value:auto-value-annotations']
|
||||
compile deps['com.google.common.html.types:types']
|
||||
compile deps['com.google.cloud:google-cloud-core']
|
||||
compile deps['com.google.cloud:google-cloud-storage']
|
||||
compile deps['com.google.guava:guava']
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.ibm.icu:icu4j:57.1
|
||||
commons-codec:commons-codec:1.11
|
||||
commons-logging:commons-logging:1.2
|
||||
@@ -47,7 +48,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
org.apache.commons:commons-lang3:3.8.1
|
||||
org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
@@ -55,9 +55,9 @@ org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:7.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.ibm.icu:icu4j:57.1
|
||||
commons-codec:commons-codec:1.11
|
||||
commons-logging:commons-logging:1.2
|
||||
@@ -47,7 +48,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
org.apache.commons:commons-lang3:3.8.1
|
||||
org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
@@ -55,9 +55,9 @@ org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:7.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.ibm.icu:icu4j:57.1
|
||||
commons-codec:commons-codec:1.11
|
||||
commons-logging:commons-logging:1.2
|
||||
@@ -47,7 +48,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
org.apache.commons:commons-lang3:3.8.1
|
||||
org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
@@ -55,9 +55,9 @@ org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:7.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.ibm.icu:icu4j:57.1
|
||||
commons-codec:commons-codec:1.11
|
||||
commons-logging:commons-logging:1.2
|
||||
@@ -47,7 +48,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
org.apache.commons:commons-lang3:3.8.1
|
||||
org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
@@ -55,9 +55,9 @@ org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:7.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.ibm.icu:icu4j:57.1
|
||||
commons-codec:commons-codec:1.11
|
||||
commons-logging:commons-logging:1.2
|
||||
@@ -47,7 +48,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
org.apache.commons:commons-lang3:3.8.1
|
||||
org.apache.commons:commons-text:1.6
|
||||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
@@ -55,9 +55,9 @@ org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:7.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
com.google.truth:truth:1.1.2
|
||||
com.ibm.icu:icu4j:57.1
|
||||
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
junit:junit:4.13.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
|
||||
org.mockito:mockito-core:3.7.7
|
||||
org.objenesis:objenesis:3.1
|
||||
org.opentest4j:opentest4j:1.2.0
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:9.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
com.google.truth:truth:1.1.2
|
||||
com.ibm.icu:icu4j:57.1
|
||||
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
junit:junit:4.13.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
|
||||
org.mockito:mockito-core:3.7.7
|
||||
org.objenesis:objenesis:3.1
|
||||
org.opentest4j:opentest4j:1.2.0
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:9.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
com.google.truth:truth:1.1.2
|
||||
com.ibm.icu:icu4j:57.1
|
||||
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
junit:junit:4.13.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
|
||||
org.mockito:mockito-core:3.7.7
|
||||
org.objenesis:objenesis:3.1
|
||||
org.opentest4j:opentest4j:1.2.0
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:9.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
com.google.truth:truth:1.1.2
|
||||
com.ibm.icu:icu4j:57.1
|
||||
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
junit:junit:4.13.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
|
||||
org.mockito:mockito-core:3.7.7
|
||||
org.objenesis:objenesis:3.1
|
||||
org.opentest4j:opentest4j:1.2.0
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:9.0
|
||||
org.threeten:threetenbp:1.5.0
|
||||
|
||||
@@ -23,6 +23,7 @@ import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.html.types.TrustedResourceUrls;
|
||||
import com.google.template.soy.SoyFileSet;
|
||||
import com.google.template.soy.tofu.SoyTofu;
|
||||
import google.registry.gradle.plugin.ProjectData.TaskData;
|
||||
@@ -118,7 +119,7 @@ final class CoverPageGenerator {
|
||||
|
||||
builder.put("projectState", state.toString());
|
||||
builder.put("title", title);
|
||||
builder.put("cssFiles", ImmutableSet.of("css/style.css"));
|
||||
builder.put("cssFiles", ImmutableSet.of(TrustedResourceUrls.fromConstant("css/style.css")));
|
||||
builder.put("invocation", getInvocation());
|
||||
builder.put("tasksByState", getTasksByStateSoyData());
|
||||
return builder.build();
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
{template .coverPage}
|
||||
{@param title: string}
|
||||
{@param cssFiles: list<string>}
|
||||
{@param cssFiles: list<trusted_resource_uri>}
|
||||
{@param projectState: string}
|
||||
{@param invocation: string}
|
||||
{@param tasksByState: map<string, list<[uniqueName: string, description: string, log: string, reports: map<string, string>]>>}
|
||||
|
||||
@@ -81,7 +81,7 @@ PRESUBMITS = {
|
||||
".git", "/build/", "/generated/", "/generated_tests/",
|
||||
"node_modules/", "JUnitBackports.java", "registrar_bin.",
|
||||
"registrar_dbg.", "google-java-format-diff.py",
|
||||
"nomulus.golden.sql", "soyutils_usegoog.js"
|
||||
"nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
|
||||
}, REQUIRED):
|
||||
"File did not include the license header.",
|
||||
|
||||
@@ -213,6 +213,7 @@ PRESUBMITS = {
|
||||
"RdapDomainSearchAction.java",
|
||||
"RdapNameserverSearchAction.java",
|
||||
"RdapSearchActionBase.java",
|
||||
"RegistryQuery",
|
||||
},
|
||||
):
|
||||
"The first String parameter to EntityManager.create(Native)Query "
|
||||
|
||||
@@ -317,6 +317,7 @@ dependencies {
|
||||
testCompile deps['com.google.monitoring-client:contrib']
|
||||
testCompile deps['com.google.truth:truth']
|
||||
testCompile deps['com.google.truth.extensions:truth-java8-extension']
|
||||
testCompile deps['org.checkerframework:checker-qual']
|
||||
testCompile deps['org.hamcrest:hamcrest']
|
||||
testCompile deps['org.hamcrest:hamcrest-core']
|
||||
testCompile deps['org.hamcrest:hamcrest-library']
|
||||
@@ -330,7 +331,6 @@ dependencies {
|
||||
testCompile deps['org.junit.platform:junit-platform-suite-api']
|
||||
testCompile deps['org.mockito:mockito-core']
|
||||
testCompile deps['org.mockito:mockito-junit-jupiter']
|
||||
testCompile 'org.checkerframework:checker-qual:3.9.1'
|
||||
runtime deps['org.postgresql:postgresql']
|
||||
|
||||
// Indirect dependency found by undeclared-dependency check. Such
|
||||
@@ -433,12 +433,9 @@ task soyToJava {
|
||||
// Relative paths of soy directories.
|
||||
def spec11SoyDir = "google/registry/reporting/spec11/soy"
|
||||
def toolsSoyDir = "google/registry/tools/soy"
|
||||
def uiSoyDir = "google/registry/ui/soy"
|
||||
def registrarSoyDir = "google/registry/ui/soy/registrar"
|
||||
|
||||
def soyRelativeDirs = [
|
||||
spec11SoyDir, toolsSoyDir, uiSoyDir, registrarSoyDir,
|
||||
]
|
||||
def soyRelativeDirs = [spec11SoyDir, toolsSoyDir, registrarSoyDir]
|
||||
soyRelativeDirs.each {
|
||||
inputs.dir "${resourcesSourceDir}/${it}"
|
||||
outputs.dir "${generatedDir}/${it}"
|
||||
@@ -452,7 +449,8 @@ task soyToJava {
|
||||
"--outputDirectory", "${outputDirectory}",
|
||||
"--javaClassNameSource", "filename",
|
||||
"--allowExternalCalls", "true",
|
||||
"--srcs", "${soyFiles.join(',')}"
|
||||
"--srcs", "${soyFiles.join(',')}",
|
||||
"--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -469,14 +467,6 @@ task soyToJava {
|
||||
dir: "${resourcesSourceDir}/${registrarSoyDir}",
|
||||
include: ['**/*.soy']))
|
||||
|
||||
soyToJava('google.registry.ui.soy',
|
||||
"${generatedDir}/${uiSoyDir}",
|
||||
files {
|
||||
file("${resourcesSourceDir}/${uiSoyDir}").listFiles()
|
||||
}.filter {
|
||||
it.name.endsWith(".soy")
|
||||
})
|
||||
|
||||
soyToJava('google.registry.reporting.spec11.soy',
|
||||
"${generatedDir}/${spec11SoyDir}",
|
||||
fileTree(
|
||||
@@ -485,42 +475,24 @@ task soyToJava {
|
||||
}
|
||||
}
|
||||
|
||||
task soyToJS {
|
||||
def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy"
|
||||
def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy"
|
||||
task soyToJS(type: JavaExec) {
|
||||
def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy/registrar"
|
||||
def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy/registrar"
|
||||
inputs.dir rootSoyDirectory
|
||||
outputs.dir outputSoyDirectory
|
||||
|
||||
ext.soyToJS = { outputDirectory, soyFiles , deps->
|
||||
javaexec {
|
||||
main = "com.google.template.soy.SoyToJsSrcCompiler"
|
||||
classpath configurations.soy
|
||||
def inputSoyFiles = files {
|
||||
file("${rootSoyDirectory}").listFiles()
|
||||
}.filter {
|
||||
it.name.endsWith(".soy")
|
||||
}
|
||||
|
||||
args "--outputPathFormat", "${outputDirectory}/{INPUT_FILE_NAME}.js",
|
||||
classpath configurations.soy
|
||||
main = "com.google.template.soy.SoyToJsSrcCompiler"
|
||||
args "--outputPathFormat", "${outputSoyDirectory}/{INPUT_FILE_NAME}.js",
|
||||
"--allowExternalCalls", "false",
|
||||
"--srcs", "${soyFiles.join(',')}",
|
||||
"--shouldProvideRequireSoyNamespaces", "true",
|
||||
"--srcs", "${inputSoyFiles.join(',')}",
|
||||
"--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
|
||||
if (deps != "") {
|
||||
args "--deps", "${deps.join(',')}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
doLast {
|
||||
def rootSoyFiles =
|
||||
fileTree(
|
||||
dir: "${rootSoyDirectory}",
|
||||
include: ['*.soy'])
|
||||
|
||||
soyToJS("${outputSoyDirectory}", rootSoyFiles, "")
|
||||
soyToJS("${outputSoyDirectory}/registrar",
|
||||
files {
|
||||
file("${rootSoyDirectory}/registrar").listFiles()
|
||||
}.filter {
|
||||
it.name.endsWith(".soy")
|
||||
}, rootSoyFiles)
|
||||
}
|
||||
}
|
||||
|
||||
task stylesheetsToJavascript {
|
||||
@@ -603,8 +575,8 @@ task compileProdJS(type: JavaExec) {
|
||||
closureArgs << "--generate_exports"
|
||||
|
||||
// manually include all the required js files
|
||||
closureArgs << "--js=${nodeModulesDir}/google-closure-library/**.js"
|
||||
closureArgs << "--js=${jsDir}/soyutils_usegoog.js"
|
||||
closureArgs << "--js=${nodeModulesDir}/google-closure-library/**/*.js"
|
||||
closureArgs << "--js=${jsDir}/*.js"
|
||||
closureArgs << "--js=${cssSourceDir}/registrar_bin.css.js"
|
||||
closureArgs << "--js=${jsSourceDir}/**.js"
|
||||
closureArgs << "--js=${externsDir}/json.js"
|
||||
@@ -631,15 +603,6 @@ compileProdJS.dependsOn processResources
|
||||
compileProdJS.dependsOn processTestResources
|
||||
compileProdJS.dependsOn soyToJS
|
||||
|
||||
task karmaTest(type: Exec) {
|
||||
dependsOn ':npmInstall'
|
||||
workingDir rootProject.projectDir
|
||||
executable '.gradle/nodejs/node-v14.15.5-linux-x64/bin/node'
|
||||
args('node_modules/karma/bin/karma', 'start', "${project.projectDir}/karma.conf.js")
|
||||
}
|
||||
|
||||
test.dependsOn karmaTest
|
||||
|
||||
// Make testing artifacts available to be depended up on by other projects.
|
||||
// TODO: factor out google.registry.testing to be a separate project.
|
||||
task testJar(type: Jar) {
|
||||
|
||||
@@ -1,15 +1,4 @@
|
||||
# This is a Gradle generated file for dependency locking.
|
||||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
args4j:args4j:2.0.26
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.7
|
||||
com.google.errorprone:error_prone_annotations:2.3.1
|
||||
com.google.guava:guava:25.1-jre
|
||||
com.google.j2objc:j2objc-annotations:1.1
|
||||
com.google.javascript:closure-compiler-externs:v20190301
|
||||
com.google.javascript:closure-compiler:v20190301
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.0
|
||||
com.google.protobuf:protobuf-java:3.0.2
|
||||
org.checkerframework:checker-qual:2.0.0
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.14
|
||||
com.google.javascript:closure-compiler:v20210505
|
||||
|
||||
@@ -105,9 +105,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -135,7 +136,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -104,9 +104,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
@@ -133,7 +134,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -110,9 +110,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -140,7 +141,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -110,9 +110,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -140,7 +141,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -105,9 +105,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -135,7 +136,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -104,9 +104,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
@@ -133,7 +134,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -109,9 +109,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -139,7 +140,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -109,9 +109,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -139,7 +140,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -109,9 +109,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -139,7 +140,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -110,9 +110,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -140,7 +141,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:68.2
|
||||
|
||||
@@ -5,25 +5,25 @@ aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.7
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.8.0-beta1
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:5.0.1
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:1.0.1
|
||||
com.google.protobuf:protobuf-java:3.13.0
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.ibm.icu:icu4j:57.1
|
||||
javax.annotation:jsr250-api:1.0
|
||||
javax.inject:javax.inject:1
|
||||
javax.validation:validation-api:1.0.0.GA
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:6.0
|
||||
org.ow2.asm:asm-commons:6.0
|
||||
org.ow2.asm:asm-tree:6.0
|
||||
org.ow2.asm:asm-util:6.0
|
||||
org.ow2.asm:asm:6.0
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
org.ow2.asm:asm-tree:7.0
|
||||
org.ow2.asm:asm-util:7.0
|
||||
org.ow2.asm:asm:7.0
|
||||
|
||||
@@ -106,9 +106,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -138,7 +139,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
com.google.truth:truth:1.1.2
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
||||
@@ -105,9 +105,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
@@ -136,7 +137,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
com.google.truth:truth:1.1.2
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
||||
@@ -111,9 +111,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -143,7 +144,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
com.google.truth:truth:1.1.2
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
||||
@@ -111,9 +111,10 @@ com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.4
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
@@ -143,7 +144,7 @@ com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2018-03-14
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
com.google.truth:truth:1.1.2
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
process.env.CHROME_BIN = require('puppeteer').executablePath()
|
||||
|
||||
module.exports = function(config) {
|
||||
config.set({
|
||||
basePath: '..',
|
||||
browsers: ['ChromeHeadlessNoSandbox'],
|
||||
customLaunchers: {
|
||||
ChromeHeadlessNoSandbox: {
|
||||
base: 'ChromeHeadless',
|
||||
flags: ['--no-sandbox']
|
||||
}
|
||||
},
|
||||
frameworks: ['jasmine', 'closure'],
|
||||
singleRun: true,
|
||||
autoWatch: false,
|
||||
files: [
|
||||
'node_modules/google-closure-library/closure/goog/base.js',
|
||||
'core/src/test/javascript/**/*_test.js',
|
||||
{
|
||||
pattern: 'core/src/test/javascript/**/!(*_test).js',
|
||||
included: false
|
||||
},
|
||||
{
|
||||
pattern: 'core/src/main/javascript/**/*.js',
|
||||
included: false
|
||||
},
|
||||
{
|
||||
pattern: 'core/build/generated/sources/custom/java/main/**/*.soy.js',
|
||||
included: false
|
||||
},
|
||||
{
|
||||
pattern: 'node_modules/google-closure-library/closure/goog/deps.js',
|
||||
included: false,
|
||||
served: false
|
||||
},
|
||||
{
|
||||
pattern: 'node_modules/google-closure-library/closure/goog/**/*.js',
|
||||
included: false
|
||||
},
|
||||
{
|
||||
pattern: 'core/build/resources/main/google/registry/ui/assets/images/*.png',
|
||||
included: false
|
||||
},
|
||||
{
|
||||
pattern: 'core/build/resources/main/google/registry/ui/assets/images/icons/svg/*.svg',
|
||||
included: false
|
||||
}
|
||||
],
|
||||
preprocessors: {
|
||||
'node_modules/google-closure-library/closure/goog/deps.js': ['closure', 'closure-deps'],
|
||||
'node_modules/google-closure-library/closure/goog/base.js': ['closure'],
|
||||
'node_modules/google-closure-library/closure/**/*.js': ['closure'],
|
||||
'core/src/*/javascript/**/*.js': ['closure'],
|
||||
'core/build/generated/sources/custom/java/main/**/*.soy.js': ['closure'],
|
||||
},
|
||||
proxies: {
|
||||
"/assets/": "/base/core/build/resources/main/google/registry/ui/assets/"
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
|
||||
import com.google.appengine.api.datastore.EntityTranslator;
|
||||
import com.google.common.collect.AbstractIterator;
|
||||
@@ -45,7 +45,7 @@ public class BackupUtils {
|
||||
* {@link OutputStream} in delimited protocol buffer format.
|
||||
*/
|
||||
static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
|
||||
EntityTranslator.convertToPb(ofy().save().toEntity(entity)).writeDelimitedTo(stream);
|
||||
EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -61,11 +61,12 @@ public class BackupUtils {
|
||||
@Override
|
||||
protected ImmutableObject computeNext() {
|
||||
EntityProto proto = new EntityProto();
|
||||
if (proto.parseDelimitedFrom(input)) { // False means end of stream; other errors throw.
|
||||
return ofy().load().fromEntity(EntityTranslator.createFromPb(proto));
|
||||
if (proto.parseDelimitedFrom(input)) { // False means end of stream; other errors throw.
|
||||
return auditedOfy().load().fromEntity(EntityTranslator.createFromPb(proto));
|
||||
}
|
||||
return endOfData();
|
||||
}};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public static ImmutableList<ImmutableObject> deserializeEntities(byte[] bytes) {
|
||||
|
||||
@@ -18,7 +18,7 @@ import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
@@ -64,8 +64,7 @@ public final class CommitLogCheckpointAction implements Runnable {
|
||||
final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
|
||||
logger.atInfo().log(
|
||||
"Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
|
||||
tm()
|
||||
.transact(
|
||||
tm().transact(
|
||||
() -> {
|
||||
DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
|
||||
if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {
|
||||
@@ -73,7 +72,7 @@ public final class CommitLogCheckpointAction implements Runnable {
|
||||
"Newer checkpoint already written at time: %s", lastWrittenTime);
|
||||
return;
|
||||
}
|
||||
ofy()
|
||||
auditedOfy()
|
||||
.saveWithoutBackup()
|
||||
.entities(
|
||||
checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
|
||||
|
||||
@@ -17,7 +17,7 @@ package google.registry.backup;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static java.lang.Boolean.FALSE;
|
||||
import static java.lang.Boolean.TRUE;
|
||||
@@ -66,6 +66,8 @@ import org.joda.time.Duration;
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/deleteOldCommitLogs",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
// No longer needed in SQL. Subject to future removal.
|
||||
@Deprecated
|
||||
public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
|
||||
private static final int NUM_MAP_SHARDS = 20;
|
||||
@@ -75,9 +77,17 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject @Config("commitLogDatastoreRetention") Duration maxAge;
|
||||
@Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
|
||||
@Inject DeleteOldCommitLogsAction() {}
|
||||
|
||||
@Inject
|
||||
@Config("commitLogDatastoreRetention")
|
||||
Duration maxAge;
|
||||
|
||||
@Inject
|
||||
@Parameter(PARAM_DRY_RUN)
|
||||
boolean isDryRun;
|
||||
|
||||
@Inject
|
||||
DeleteOldCommitLogsAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
@@ -138,12 +148,12 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
// If it isn't a Key<CommitLogManifest> then it should be an EppResource, which we need to
|
||||
// load to emit the revisions.
|
||||
//
|
||||
Object object = ofy().load().key(key).now();
|
||||
Object object = auditedOfy().load().key(key).now();
|
||||
checkNotNull(object, "Received a key to a missing object. key: %s", key);
|
||||
checkState(
|
||||
object instanceof EppResource,
|
||||
"Received a key to an object that isn't EppResource nor CommitLogManifest."
|
||||
+ " Key: %s object type: %s",
|
||||
+ " Key: %s object type: %s",
|
||||
key,
|
||||
object.getClass().getName());
|
||||
|
||||
@@ -224,8 +234,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
* OK to delete this manifestKey. If even one source returns "false" (meaning "it's not OK to
|
||||
* delete this manifest") then it won't be deleted.
|
||||
*/
|
||||
static class DeleteOldCommitLogsReducer
|
||||
extends Reducer<Key<CommitLogManifest>, Boolean, Void> {
|
||||
static class DeleteOldCommitLogsReducer extends Reducer<Key<CommitLogManifest>, Boolean, Void> {
|
||||
|
||||
private static final long serialVersionUID = -4918760187627937268L;
|
||||
|
||||
@@ -241,12 +250,12 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
}
|
||||
|
||||
public abstract Status status();
|
||||
|
||||
public abstract int numDeleted();
|
||||
|
||||
static DeletionResult create(Status status, int numDeleted) {
|
||||
return
|
||||
new AutoValue_DeleteOldCommitLogsAction_DeleteOldCommitLogsReducer_DeletionResult(
|
||||
status, numDeleted);
|
||||
return new AutoValue_DeleteOldCommitLogsAction_DeleteOldCommitLogsReducer_DeletionResult(
|
||||
status, numDeleted);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,8 +266,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
|
||||
@Override
|
||||
public void reduce(
|
||||
final Key<CommitLogManifest> manifestKey,
|
||||
ReducerInput<Boolean> canDeleteVerdicts) {
|
||||
final Key<CommitLogManifest> manifestKey, ReducerInput<Boolean> canDeleteVerdicts) {
|
||||
ImmutableMultiset<Boolean> canDeleteMultiset = ImmutableMultiset.copyOf(canDeleteVerdicts);
|
||||
if (canDeleteMultiset.count(TRUE) > 1) {
|
||||
getContext().incrementCounter("commit log manifests incorrectly mapped multiple times");
|
||||
@@ -267,47 +275,54 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
getContext().incrementCounter("commit log manifests referenced multiple times");
|
||||
}
|
||||
if (canDeleteMultiset.contains(FALSE)) {
|
||||
getContext().incrementCounter(
|
||||
canDeleteMultiset.contains(TRUE)
|
||||
? "old commit log manifests still referenced"
|
||||
: "new (or nonexistent) commit log manifests referenced");
|
||||
getContext().incrementCounter(
|
||||
"EPP resource revisions handled",
|
||||
canDeleteMultiset.count(FALSE));
|
||||
getContext()
|
||||
.incrementCounter(
|
||||
canDeleteMultiset.contains(TRUE)
|
||||
? "old commit log manifests still referenced"
|
||||
: "new (or nonexistent) commit log manifests referenced");
|
||||
getContext()
|
||||
.incrementCounter("EPP resource revisions handled", canDeleteMultiset.count(FALSE));
|
||||
return;
|
||||
}
|
||||
|
||||
DeletionResult deletionResult = tm().transactNew(() -> {
|
||||
CommitLogManifest manifest = ofy().load().key(manifestKey).now();
|
||||
// It is possible that the same manifestKey was run twice, if a shard had to be restarted
|
||||
// or some weird failure. If this happens, we want to exit immediately.
|
||||
// Note that this can never happen in dryRun.
|
||||
if (manifest == null) {
|
||||
return DeletionResult.create(DeletionResult.Status.ALREADY_DELETED, 0);
|
||||
}
|
||||
// Doing a sanity check on the date. This is the only place we use the CommitLogManifest,
|
||||
// so maybe removing this test will improve performance. However, unless it's proven that
|
||||
// the performance boost is significant (and we've tested this enough to be sure it never
|
||||
// happens)- the safty of "let's not delete stuff we need from prod" is more important.
|
||||
if (manifest.getCommitTime().isAfter(deletionThreshold)) {
|
||||
return DeletionResult.create(DeletionResult.Status.AFTER_THRESHOLD, 0);
|
||||
}
|
||||
Iterable<Key<CommitLogMutation>> commitLogMutationKeys = ofy().load()
|
||||
.type(CommitLogMutation.class)
|
||||
.ancestor(manifestKey)
|
||||
.keys()
|
||||
.iterable();
|
||||
ImmutableList<Key<?>> keysToDelete = ImmutableList.<Key<?>>builder()
|
||||
.addAll(commitLogMutationKeys)
|
||||
.add(manifestKey)
|
||||
.build();
|
||||
// Normally in a dry run we would log the entities that would be deleted, but those can
|
||||
// number in the millions so we skip the logging.
|
||||
if (!isDryRun) {
|
||||
ofy().deleteWithoutBackup().keys(keysToDelete);
|
||||
}
|
||||
return DeletionResult.create(DeletionResult.Status.SUCCESS, keysToDelete.size());
|
||||
});
|
||||
DeletionResult deletionResult =
|
||||
tm().transactNew(
|
||||
() -> {
|
||||
CommitLogManifest manifest = auditedOfy().load().key(manifestKey).now();
|
||||
// It is possible that the same manifestKey was run twice, if a shard had to be
|
||||
// restarted or some weird failure. If this happens, we want to exit
|
||||
// immediately. Note that this can never happen in dryRun.
|
||||
if (manifest == null) {
|
||||
return DeletionResult.create(DeletionResult.Status.ALREADY_DELETED, 0);
|
||||
}
|
||||
// Doing a sanity check on the date. This is the only place we use the
|
||||
// CommitLogManifest, so maybe removing this test will improve performance.
|
||||
// However, unless it's proven that the performance boost is significant (and
|
||||
// we've tested this enough to be sure it never happens)- the safety of "let's
|
||||
// not delete stuff we need from prod" is more important.
|
||||
if (manifest.getCommitTime().isAfter(deletionThreshold)) {
|
||||
return DeletionResult.create(DeletionResult.Status.AFTER_THRESHOLD, 0);
|
||||
}
|
||||
Iterable<Key<CommitLogMutation>> commitLogMutationKeys =
|
||||
auditedOfy()
|
||||
.load()
|
||||
.type(CommitLogMutation.class)
|
||||
.ancestor(manifestKey)
|
||||
.keys()
|
||||
.iterable();
|
||||
ImmutableList<Key<?>> keysToDelete =
|
||||
ImmutableList.<Key<?>>builder()
|
||||
.addAll(commitLogMutationKeys)
|
||||
.add(manifestKey)
|
||||
.build();
|
||||
// Normally in a dry run we would log the entities that would be deleted, but
|
||||
// those can number in the millions so we skip the logging.
|
||||
if (!isDryRun) {
|
||||
auditedOfy().deleteWithoutBackup().keys(keysToDelete);
|
||||
}
|
||||
return DeletionResult.create(
|
||||
DeletionResult.Status.SUCCESS, keysToDelete.size());
|
||||
});
|
||||
|
||||
switch (deletionResult.status()) {
|
||||
case SUCCESS:
|
||||
|
||||
@@ -25,7 +25,7 @@ import static google.registry.backup.BackupUtils.GcsMetadataKeys.NUM_TRANSACTION
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.UPPER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.BackupUtils.serializeEntity;
|
||||
import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static java.nio.channels.Channels.newOutputStream;
|
||||
@@ -89,11 +89,14 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
checkArgument(lowerCheckpointTime.isBefore(upperCheckpointTime));
|
||||
// Load the boundary checkpoints - lower is exclusive and may not exist (on the first export,
|
||||
// when lowerCheckpointTime is START_OF_TIME), whereas the upper is inclusive and must exist.
|
||||
CommitLogCheckpoint lowerCheckpoint = lowerCheckpointTime.isAfter(START_OF_TIME)
|
||||
? verifyNotNull(ofy().load().key(CommitLogCheckpoint.createKey(lowerCheckpointTime)).now())
|
||||
: null;
|
||||
CommitLogCheckpoint lowerCheckpoint =
|
||||
lowerCheckpointTime.isAfter(START_OF_TIME)
|
||||
? verifyNotNull(
|
||||
auditedOfy().load().key(CommitLogCheckpoint.createKey(lowerCheckpointTime)).now())
|
||||
: null;
|
||||
CommitLogCheckpoint upperCheckpoint =
|
||||
verifyNotNull(ofy().load().key(CommitLogCheckpoint.createKey(upperCheckpointTime)).now());
|
||||
verifyNotNull(
|
||||
auditedOfy().load().key(CommitLogCheckpoint.createKey(upperCheckpointTime)).now());
|
||||
|
||||
// Load the keys of all the manifests to include in this diff.
|
||||
List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
|
||||
@@ -117,7 +120,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
// asynchronously load the entities for the next one.
|
||||
List<List<Key<CommitLogManifest>>> keyChunks = partition(sortedKeys, batchSize);
|
||||
// Objectify's map return type is asynchronous. Calling .values() will block until it loads.
|
||||
Map<?, CommitLogManifest> nextChunkToExport = ofy().load().keys(keyChunks.get(0));
|
||||
Map<?, CommitLogManifest> nextChunkToExport = auditedOfy().load().keys(keyChunks.get(0));
|
||||
for (int i = 0; i < keyChunks.size(); i++) {
|
||||
// Force the async load to finish.
|
||||
Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
|
||||
@@ -125,10 +128,10 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
// Since there is no hard bound on how much data this might be, take care not to let the
|
||||
// Objectify session cache fill up and potentially run out of memory. This is the only safe
|
||||
// point to do this since at this point there is no async load in progress.
|
||||
ofy().clearSessionCache();
|
||||
auditedOfy().clearSessionCache();
|
||||
// Kick off the next async load, which can happen in parallel to the current GCS export.
|
||||
if (i + 1 < keyChunks.size()) {
|
||||
nextChunkToExport = ofy().load().keys(keyChunks.get(i + 1));
|
||||
nextChunkToExport = auditedOfy().load().keys(keyChunks.get(i + 1));
|
||||
}
|
||||
exportChunk(gcsStream, chunkValues);
|
||||
logger.atInfo().log("Exported %d manifests", chunkValues.size());
|
||||
@@ -192,7 +195,8 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
return ImmutableSet.of();
|
||||
}
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(bucketNum);
|
||||
return ofy().load()
|
||||
return auditedOfy()
|
||||
.load()
|
||||
.type(CommitLogManifest.class)
|
||||
.ancestor(bucketKey)
|
||||
.filterKey(">=", CommitLogManifest.createKey(bucketKey, lowerBound))
|
||||
@@ -208,7 +212,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
||||
new ImmutableList.Builder<>();
|
||||
for (CommitLogManifest manifest : chunk) {
|
||||
entities.add(ImmutableList.of(manifest));
|
||||
entities.add(ofy().load().type(CommitLogMutation.class).ancestor(manifest));
|
||||
entities.add(auditedOfy().load().type(CommitLogMutation.class).ancestor(manifest));
|
||||
}
|
||||
for (ImmutableObject entity : concat(entities.build())) {
|
||||
serializeEntity(entity, gcsStream);
|
||||
|
||||
@@ -16,7 +16,7 @@ package google.registry.backup;
|
||||
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
|
||||
import static org.joda.time.Duration.standardHours;
|
||||
@@ -27,11 +27,14 @@ import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
|
||||
import google.registry.model.server.Lock;
|
||||
import google.registry.model.translators.VKeyTranslatorFactory;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Method;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.schema.replay.DatastoreEntity;
|
||||
@@ -39,6 +42,7 @@ import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
import google.registry.schema.replay.NonReplicatedEntity;
|
||||
import google.registry.schema.replay.ReplaySpecializer;
|
||||
import google.registry.schema.replay.SqlReplayCheckpoint;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.RequestStatusChecker;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
@@ -53,7 +57,7 @@ import org.joda.time.Duration;
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = ReplayCommitLogsToSqlAction.PATH,
|
||||
method = Action.Method.POST,
|
||||
method = Method.POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
@@ -69,15 +73,19 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
@Inject Response response;
|
||||
@Inject RequestStatusChecker requestStatusChecker;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
@Inject Clock clock;
|
||||
|
||||
@Inject
|
||||
ReplayCommitLogsToSqlAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
if (!RegistryConfig.getCloudSqlReplayCommitLogs()) {
|
||||
String message = "ReplayCommitLogsToSqlAction was called but disabled in the config.";
|
||||
logger.atWarning().log(message);
|
||||
MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc());
|
||||
if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
|
||||
String message =
|
||||
String.format(
|
||||
"Skipping ReplayCommitLogsToSqlAction because we are in migration phase %s.", state);
|
||||
logger.atInfo().log(message);
|
||||
// App Engine will retry on any non-2xx status code, which we don't want in this case.
|
||||
response.setStatus(SC_NO_CONTENT);
|
||||
response.setPayload(message);
|
||||
@@ -151,7 +159,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
}
|
||||
|
||||
private void handleEntityPut(Entity entity) {
|
||||
Object ofyPojo = ofy().toPojo(entity);
|
||||
Object ofyPojo = auditedOfy().toPojo(entity);
|
||||
if (ofyPojo instanceof DatastoreEntity) {
|
||||
DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
|
||||
datastoreEntity
|
||||
|
||||
@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterators.peekingIterator;
|
||||
import static google.registry.backup.BackupUtils.createDeserializingIterator;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
|
||||
import com.google.appengine.api.datastore.DatastoreService;
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
@@ -146,10 +146,10 @@ public class RestoreCommitLogsAction implements Runnable {
|
||||
private CommitLogManifest restoreOneTransaction(PeekingIterator<ImmutableObject> commitLogs) {
|
||||
final CommitLogManifest manifest = (CommitLogManifest) commitLogs.next();
|
||||
Result<?> deleteResult = deleteAsync(manifest.getDeletions());
|
||||
List<Entity> entitiesToSave = Lists.newArrayList(ofy().save().toEntity(manifest));
|
||||
List<Entity> entitiesToSave = Lists.newArrayList(auditedOfy().save().toEntity(manifest));
|
||||
while (commitLogs.hasNext() && commitLogs.peek() instanceof CommitLogMutation) {
|
||||
CommitLogMutation mutation = (CommitLogMutation) commitLogs.next();
|
||||
entitiesToSave.add(ofy().save().toEntity(mutation));
|
||||
entitiesToSave.add(auditedOfy().save().toEntity(mutation));
|
||||
entitiesToSave.add(EntityTranslator.createFromPbBytes(mutation.getEntityProtoBytes()));
|
||||
}
|
||||
saveRaw(entitiesToSave);
|
||||
@@ -176,7 +176,8 @@ public class RestoreCommitLogsAction implements Runnable {
|
||||
return;
|
||||
}
|
||||
retrier.callWithRetry(
|
||||
() -> ofy().saveWithoutBackup().entities(objectsToSave).now(), RuntimeException.class);
|
||||
() -> auditedOfy().saveWithoutBackup().entities(objectsToSave).now(),
|
||||
RuntimeException.class);
|
||||
}
|
||||
|
||||
private Result<?> deleteAsync(Set<Key<?>> keysToDelete) {
|
||||
@@ -185,7 +186,7 @@ public class RestoreCommitLogsAction implements Runnable {
|
||||
}
|
||||
return dryRun || keysToDelete.isEmpty()
|
||||
? new ResultNow<Void>(null)
|
||||
: ofy().deleteWithoutBackup().keys(keysToDelete);
|
||||
: auditedOfy().deleteWithoutBackup().keys(keysToDelete);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ import javax.annotation.Nullable;
|
||||
*
|
||||
* <ul>
|
||||
* <li>Convert an Objectify entity to a Datastore {@link Entity}: {@code
|
||||
* ofy().save().toEntity(..)}
|
||||
* auditedOfy().save().toEntity(..)}
|
||||
* <li>Entity is serializable, but the more efficient approach is to convert an Entity to a
|
||||
* ProtocolBuffer ({@link com.google.storage.onestore.v3.OnestoreEntity.EntityProto}) and then
|
||||
* to raw bytes.
|
||||
|
||||
@@ -34,7 +34,7 @@ import static google.registry.model.ResourceTransferUtils.denyPendingTransfer;
|
||||
import static google.registry.model.ResourceTransferUtils.handlePendingTransferOnDelete;
|
||||
import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
|
||||
import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE_FAILURE;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE;
|
||||
@@ -336,7 +336,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
||||
DeletionRequest deletionRequest, boolean hasNoActiveReferences) {
|
||||
DateTime now = tm().getTransactionTime();
|
||||
EppResource resource =
|
||||
ofy().load().key(deletionRequest.key()).now().cloneProjectedAtTime(now);
|
||||
auditedOfy().load().key(deletionRequest.key()).now().cloneProjectedAtTime(now);
|
||||
// Double-check transactionally that the resource is still active and in PENDING_DELETE.
|
||||
if (!doesResourceStateAllowDeletion(resource, now)) {
|
||||
return DeletionResult.create(Type.ERRORED, "");
|
||||
@@ -370,11 +370,10 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
||||
: "it was transferred prior to deletion");
|
||||
|
||||
HistoryEntry historyEntry =
|
||||
new HistoryEntry.Builder()
|
||||
HistoryEntry.createBuilderForResource(resource)
|
||||
.setClientId(deletionRequest.requestingClientId())
|
||||
.setModificationTime(now)
|
||||
.setType(getHistoryEntryType(resource, deleteAllowed))
|
||||
.setParent(deletionRequest.key())
|
||||
.build();
|
||||
|
||||
PollMessage.OneTime pollMessage =
|
||||
@@ -409,7 +408,9 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
||||
} else {
|
||||
resourceToSave = resource.asBuilder().removeStatusValue(PENDING_DELETE).build();
|
||||
}
|
||||
ofy().save().<ImmutableObject>entities(resourceToSave, historyEntry, pollMessage);
|
||||
auditedOfy()
|
||||
.save()
|
||||
.<ImmutableObject>entities(resourceToSave, historyEntry.asHistoryEntry(), pollMessage);
|
||||
return DeletionResult.create(
|
||||
deleteAllowed ? Type.DELETED : Type.NOT_DELETED, pollMessageText);
|
||||
}
|
||||
@@ -526,7 +527,8 @@ public class DeleteContactsAndHostsAction implements Runnable {
|
||||
Key.create(
|
||||
checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
|
||||
EppResource resource =
|
||||
checkNotNull(ofy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
|
||||
checkNotNull(
|
||||
auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
|
||||
checkState(
|
||||
resource instanceof ContactResource || resource instanceof HostResource,
|
||||
"Cannot delete a %s via this action",
|
||||
|
||||
@@ -17,8 +17,8 @@ package google.registry.batch;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static google.registry.flows.FlowUtils.marshalWithLenientRetry;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
import static google.registry.util.ResourceUtils.readResourceUtf8;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
@@ -36,6 +36,7 @@ import google.registry.flows.StatelessRequestSessionMetadata;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.eppcommon.ProtocolDefinition;
|
||||
import google.registry.model.eppoutput.EppOutput;
|
||||
import google.registry.persistence.transaction.QueryComposer.Comparator;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
@@ -128,12 +129,15 @@ public class DeleteExpiredDomainsAction implements Runnable {
|
||||
logger.atInfo().log(
|
||||
"Deleting non-renewing domains with autorenew end times up through %s.", runTime);
|
||||
|
||||
// Note: This query is (and must be) non-transactional, and thus, is only eventually consistent.
|
||||
// Note: in Datastore, this query is (and must be) non-transactional, and thus, is only
|
||||
// eventually consistent.
|
||||
ImmutableList<DomainBase> domainsToDelete =
|
||||
ofy().load().type(DomainBase.class).filter("autorenewEndTime <=", runTime).list().stream()
|
||||
// Datastore can't do two inequalities in one query, so the second happens in-memory.
|
||||
.filter(d -> d.getDeletionTime().isEqual(END_OF_TIME))
|
||||
.collect(toImmutableList());
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
tm().createQueryComposer(DomainBase.class)
|
||||
.where("autorenewEndTime", Comparator.LTE, runTime)
|
||||
.where("deletionTime", Comparator.EQ, END_OF_TIME)
|
||||
.list());
|
||||
if (domainsToDelete.isEmpty()) {
|
||||
logger.atInfo().log("Found 0 domains to delete.");
|
||||
response.setPayload("Found 0 domains to delete.");
|
||||
|
||||
@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkState;
|
||||
import static google.registry.config.RegistryEnvironment.PRODUCTION;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
@@ -125,12 +125,11 @@ public class DeleteLoadTestDataAction implements Runnable {
|
||||
Key.create(EppResourceIndex.create(Key.create(resource)));
|
||||
final Key<? extends ForeignKeyIndex<?>> fki = ForeignKeyIndex.createKey(resource);
|
||||
int numEntitiesDeleted =
|
||||
tm()
|
||||
.transact(
|
||||
tm().transact(
|
||||
() -> {
|
||||
// This ancestor query selects all descendant entities.
|
||||
List<Key<Object>> resourceAndDependentKeys =
|
||||
ofy().load().ancestor(resource).keys().list();
|
||||
auditedOfy().load().ancestor(resource).keys().list();
|
||||
ImmutableSet<Key<?>> allKeys =
|
||||
new ImmutableSet.Builder<Key<?>>()
|
||||
.add(fki)
|
||||
@@ -140,7 +139,7 @@ public class DeleteLoadTestDataAction implements Runnable {
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log("Would hard-delete the following entities: %s", allKeys);
|
||||
} else {
|
||||
ofy().deleteWithoutBackup().keys(allKeys);
|
||||
auditedOfy().deleteWithoutBackup().keys(allKeys);
|
||||
}
|
||||
return allKeys.size();
|
||||
});
|
||||
|
||||
@@ -20,7 +20,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.config.RegistryEnvironment.PRODUCTION;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.model.registry.Registries.getTldsOfType;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
@@ -44,11 +44,11 @@ import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.model.EppResourceUtils;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
@@ -166,7 +166,7 @@ public class DeleteProberDataAction implements Runnable {
|
||||
}
|
||||
|
||||
private void deleteDomain(final Key<DomainBase> domainKey) {
|
||||
final DomainBase domain = ofy().load().key(domainKey).now();
|
||||
final DomainBase domain = auditedOfy().load().key(domainKey).now();
|
||||
|
||||
DateTime now = DateTime.now(UTC);
|
||||
|
||||
@@ -220,14 +220,13 @@ public class DeleteProberDataAction implements Runnable {
|
||||
final Key<? extends ForeignKeyIndex<?>> fki = ForeignKeyIndex.createKey(domain);
|
||||
|
||||
int entitiesDeleted =
|
||||
tm()
|
||||
.transact(
|
||||
tm().transact(
|
||||
() -> {
|
||||
// This ancestor query selects all descendant HistoryEntries, BillingEvents,
|
||||
// PollMessages,
|
||||
// and TLD-specific entities, as well as the domain itself.
|
||||
List<Key<Object>> domainAndDependentKeys =
|
||||
ofy().load().ancestor(domainKey).keys().list();
|
||||
auditedOfy().load().ancestor(domainKey).keys().list();
|
||||
ImmutableSet<Key<?>> allKeys =
|
||||
new ImmutableSet.Builder<Key<?>>()
|
||||
.add(fki)
|
||||
@@ -237,7 +236,7 @@ public class DeleteProberDataAction implements Runnable {
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log("Would hard-delete the following entities: %s", allKeys);
|
||||
} else {
|
||||
ofy().deleteWithoutBackup().keys(allKeys);
|
||||
auditedOfy().deleteWithoutBackup().keys(allKeys);
|
||||
}
|
||||
return allKeys.size();
|
||||
});
|
||||
@@ -254,9 +253,9 @@ public class DeleteProberDataAction implements Runnable {
|
||||
.setDeletionTime(tm().getTransactionTime())
|
||||
.setStatusValues(null)
|
||||
.build();
|
||||
HistoryEntry historyEntry =
|
||||
new HistoryEntry.Builder()
|
||||
.setParent(domain)
|
||||
DomainHistory historyEntry =
|
||||
new DomainHistory.Builder()
|
||||
.setDomain(domain)
|
||||
.setType(DOMAIN_DELETE)
|
||||
.setModificationTime(tm().getTransactionTime())
|
||||
.setBySuperuser(true)
|
||||
@@ -264,11 +263,9 @@ public class DeleteProberDataAction implements Runnable {
|
||||
.setClientId(registryAdminClientId)
|
||||
.build();
|
||||
// Note that we don't bother handling grace periods, billing events, pending
|
||||
// transfers,
|
||||
// poll messages, or auto-renews because these will all be hard-deleted the next
|
||||
// time the
|
||||
// mapreduce runs anyway.
|
||||
ofy().save().entities(deletedDomain, historyEntry);
|
||||
// transfers, poll messages, or auto-renews because these will all be hard-deleted
|
||||
// the next time the mapreduce runs anyway.
|
||||
tm().putAll(deletedDomain, historyEntry);
|
||||
updateForeignKeyIndexDeletionTime(deletedDomain);
|
||||
dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
|
||||
});
|
||||
|
||||
@@ -21,9 +21,12 @@ import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createChildEntityInput;
|
||||
import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
|
||||
import static google.registry.model.domain.Period.Unit.YEARS;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
|
||||
import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
|
||||
import static google.registry.util.CollectionUtils.union;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
@@ -38,10 +41,8 @@ import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Range;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.NullInput;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
@@ -49,11 +50,12 @@ import google.registry.model.billing.BillingEvent.OneTime;
|
||||
import google.registry.model.billing.BillingEvent.Recurring;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.Period;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.reporting.DomainTransactionRecord;
|
||||
import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
@@ -91,30 +93,88 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
|
||||
DateTime executeTime = clock.nowUtc();
|
||||
DateTime persistedCursorTime = (cursor == null ? START_OF_TIME : cursor.getCursorTime());
|
||||
DateTime persistedCursorTime =
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
|
||||
.orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
|
||||
.getCursorTime());
|
||||
DateTime cursorTime = cursorTimeParam.orElse(persistedCursorTime);
|
||||
checkArgument(
|
||||
cursorTime.isBefore(executeTime),
|
||||
"Cursor time must be earlier than execution time.");
|
||||
cursorTime.isBefore(executeTime), "Cursor time must be earlier than execution time.");
|
||||
logger.atInfo().log(
|
||||
"Running Recurring billing event expansion for billing time range [%s, %s).",
|
||||
cursorTime, executeTime);
|
||||
mrRunner
|
||||
.setJobName("Expand Recurring billing events into synthetic OneTime events.")
|
||||
.setModuleName("backend")
|
||||
.runMapreduce(
|
||||
new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
|
||||
new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
|
||||
// Add an extra shard that maps over a null recurring event (see the mapper for why).
|
||||
ImmutableList.of(
|
||||
new NullInput<>(),
|
||||
createChildEntityInput(
|
||||
ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
}
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Expand Recurring billing events into synthetic OneTime events.")
|
||||
.setModuleName("backend")
|
||||
.runMapreduce(
|
||||
new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
|
||||
new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
|
||||
// Add an extra shard that maps over a null recurring event (see the mapper for why).
|
||||
ImmutableList.of(
|
||||
new NullInput<>(),
|
||||
createChildEntityInput(
|
||||
ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
int numBillingEventsSaved =
|
||||
jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm()
|
||||
.query(
|
||||
"FROM BillingRecurrence "
|
||||
+ "WHERE eventTime <= :executeTime "
|
||||
+ "AND eventTime < recurrenceEndTime "
|
||||
+ "ORDER BY id ASC",
|
||||
Recurring.class)
|
||||
.setParameter("executeTime", executeTime)
|
||||
// Need to get a list from the transaction and then convert it to a stream
|
||||
// for further processing. If we get a stream directly, each elements gets
|
||||
// processed downstream eagerly but Hibernate returns a
|
||||
// ScrollableResultsIterator that cannot be advanced outside the
|
||||
// transaction, resulting in an exception.
|
||||
.getResultList())
|
||||
.stream()
|
||||
.map(
|
||||
recurring ->
|
||||
jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
expandBillingEvent(recurring, executeTime, cursorTime, isDryRun)))
|
||||
.reduce(0, Integer::sum);
|
||||
|
||||
if (!isDryRun) {
|
||||
logger.atInfo().log("Saved OneTime billing events", numBillingEventsSaved);
|
||||
} else {
|
||||
logger.atInfo().log("Generated OneTime billing events (dry run)", numBillingEventsSaved);
|
||||
}
|
||||
logger.atInfo().log(
|
||||
"Recurring event expansion %s complete for billing event range [%s, %s).",
|
||||
isDryRun ? "(dry run) " : "", cursorTime, executeTime);
|
||||
tm().transact(
|
||||
() -> {
|
||||
// Check for the unlikely scenario where the cursor has been altered during the
|
||||
// expansion.
|
||||
DateTime currentCursorTime =
|
||||
tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
|
||||
.orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
|
||||
.getCursorTime();
|
||||
if (!currentCursorTime.equals(persistedCursorTime)) {
|
||||
throw new IllegalStateException(
|
||||
String.format(
|
||||
"Current cursor position %s does not match persisted cursor position %s.",
|
||||
currentCursorTime, persistedCursorTime));
|
||||
}
|
||||
if (!isDryRun) {
|
||||
tm().put(Cursor.createGlobal(RECURRING_BILLING, executeTime));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
/** Mapper to expand {@link Recurring} billing events into synthetic {@link OneTime} events. */
|
||||
public static class ExpandRecurringBillingEventsMapper
|
||||
extends Mapper<Recurring, DateTime, DateTime> {
|
||||
@@ -153,98 +213,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
try {
|
||||
numBillingEventsSaved =
|
||||
tm().transactNew(
|
||||
() -> {
|
||||
ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder =
|
||||
new ImmutableSet.Builder<>();
|
||||
final Registry tld =
|
||||
Registry.get(getTldFromDomainName(recurring.getTargetId()));
|
||||
|
||||
// Determine the complete set of times at which this recurring event should
|
||||
// occur (up to and including the runtime of the mapreduce).
|
||||
Iterable<DateTime> eventTimes =
|
||||
recurring
|
||||
.getRecurrenceTimeOfYear()
|
||||
.getInstancesInRange(
|
||||
Range.closed(
|
||||
recurring.getEventTime(),
|
||||
earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
|
||||
|
||||
// Convert these event times to billing times
|
||||
final ImmutableSet<DateTime> billingTimes =
|
||||
getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
|
||||
|
||||
Key<? extends EppResource> domainKey = recurring.getParentKey().getParent();
|
||||
Iterable<OneTime> oneTimesForDomain =
|
||||
ofy().load().type(OneTime.class).ancestor(domainKey);
|
||||
|
||||
// Determine the billing times that already have OneTime events persisted.
|
||||
ImmutableSet<DateTime> existingBillingTimes =
|
||||
getExistingBillingTimes(oneTimesForDomain, recurring);
|
||||
|
||||
ImmutableSet.Builder<HistoryEntry> historyEntriesBuilder =
|
||||
new ImmutableSet.Builder<>();
|
||||
// Create synthetic OneTime events for all billing times that do not yet have
|
||||
// an event persisted.
|
||||
for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
|
||||
// Construct a new HistoryEntry that parents over the OneTime
|
||||
HistoryEntry historyEntry =
|
||||
new HistoryEntry.Builder()
|
||||
.setBySuperuser(false)
|
||||
.setClientId(recurring.getClientId())
|
||||
.setModificationTime(tm().getTransactionTime())
|
||||
.setParent(domainKey)
|
||||
.setPeriod(Period.create(1, YEARS))
|
||||
.setReason(
|
||||
"Domain autorenewal by ExpandRecurringBillingEventsAction")
|
||||
.setRequestedByRegistrar(false)
|
||||
.setType(DOMAIN_AUTORENEW)
|
||||
// Don't write a domain transaction record if the recurrence was
|
||||
// ended prior to the billing time (i.e. a domain was deleted
|
||||
// during the autorenew grace period).
|
||||
.setDomainTransactionRecords(
|
||||
recurring.getRecurrenceEndTime().isBefore(billingTime)
|
||||
? ImmutableSet.of()
|
||||
: ImmutableSet.of(
|
||||
DomainTransactionRecord.create(
|
||||
tld.getTldStr(),
|
||||
// We report this when the autorenew grace period
|
||||
// ends
|
||||
billingTime,
|
||||
TransactionReportField.netRenewsFieldFromYears(1),
|
||||
1)))
|
||||
.build();
|
||||
historyEntriesBuilder.add(historyEntry);
|
||||
|
||||
DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
|
||||
// Determine the cost for a one-year renewal.
|
||||
Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
|
||||
syntheticOneTimesBuilder.add(
|
||||
new OneTime.Builder()
|
||||
.setBillingTime(billingTime)
|
||||
.setClientId(recurring.getClientId())
|
||||
.setCost(renewCost)
|
||||
.setEventTime(eventTime)
|
||||
.setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
|
||||
.setParent(historyEntry)
|
||||
.setPeriodYears(1)
|
||||
.setReason(recurring.getReason())
|
||||
.setSyntheticCreationTime(executeTime)
|
||||
.setCancellationMatchingBillingEvent(recurring.createVKey())
|
||||
.setTargetId(recurring.getTargetId())
|
||||
.build());
|
||||
}
|
||||
Set<HistoryEntry> historyEntries = historyEntriesBuilder.build();
|
||||
Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
|
||||
if (!isDryRun) {
|
||||
ImmutableSet<ImmutableObject> entitiesToSave =
|
||||
new ImmutableSet.Builder<ImmutableObject>()
|
||||
.addAll(historyEntries)
|
||||
.addAll(syntheticOneTimes)
|
||||
.build();
|
||||
ofy().save().entities(entitiesToSave).now();
|
||||
}
|
||||
return syntheticOneTimes.size();
|
||||
});
|
||||
() -> expandBillingEvent(recurring, executeTime, cursorTime, isDryRun));
|
||||
} catch (Throwable t) {
|
||||
getContext().incrementCounter("error: " + t.getClass().getSimpleName());
|
||||
getContext().incrementCounter(ERROR_COUNTER);
|
||||
@@ -256,45 +225,12 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
if (!isDryRun) {
|
||||
getContext().incrementCounter("Saved OneTime billing events", numBillingEventsSaved);
|
||||
} else {
|
||||
getContext().incrementCounter(
|
||||
"Generated OneTime billing events (dry run)", numBillingEventsSaved);
|
||||
getContext()
|
||||
.incrementCounter("Generated OneTime billing events (dry run)", numBillingEventsSaved);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters a set of {@link DateTime}s down to event times that are in scope for a particular
|
||||
* mapreduce run, given the cursor time and the mapreduce execution time.
|
||||
*/
|
||||
private ImmutableSet<DateTime> getBillingTimesInScope(
|
||||
Iterable<DateTime> eventTimes,
|
||||
DateTime cursorTime,
|
||||
DateTime executeTime,
|
||||
final Registry tld) {
|
||||
return Streams.stream(eventTimes)
|
||||
.map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
|
||||
.filter(Range.closedOpen(cursorTime, executeTime))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted
|
||||
* for a given recurring billing event.
|
||||
*/
|
||||
private ImmutableSet<DateTime> getExistingBillingTimes(
|
||||
Iterable<BillingEvent.OneTime> oneTimesForDomain,
|
||||
final BillingEvent.Recurring recurringEvent) {
|
||||
return Streams.stream(oneTimesForDomain)
|
||||
.filter(
|
||||
billingEvent ->
|
||||
recurringEvent
|
||||
.createVKey()
|
||||
.equals(billingEvent.getCancellationMatchingBillingEvent()))
|
||||
.map(OneTime::getBillingTime)
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* "Reducer" to advance the cursor after all map jobs have been completed. The NullInput into the
|
||||
* mapper will cause the mapper to emit one timestamp pair (current cursor and execution time),
|
||||
@@ -327,7 +263,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
isDryRun ? "(dry run) " : "", cursorTime, executionTime);
|
||||
tm().transact(
|
||||
() -> {
|
||||
Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
|
||||
Cursor cursor =
|
||||
auditedOfy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
|
||||
DateTime currentCursorTime =
|
||||
(cursor == null ? START_OF_TIME : cursor.getCursorTime());
|
||||
if (!currentCursorTime.equals(expectedPersistedCursorTime)) {
|
||||
@@ -342,4 +279,135 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static int expandBillingEvent(
|
||||
Recurring recurring, DateTime executeTime, DateTime cursorTime, boolean isDryRun) {
|
||||
ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder = new ImmutableSet.Builder<>();
|
||||
final Registry tld = Registry.get(getTldFromDomainName(recurring.getTargetId()));
|
||||
|
||||
// Determine the complete set of times at which this recurring event should
|
||||
// occur (up to and including the runtime of the mapreduce).
|
||||
Iterable<DateTime> eventTimes =
|
||||
recurring
|
||||
.getRecurrenceTimeOfYear()
|
||||
.getInstancesInRange(
|
||||
Range.closed(
|
||||
recurring.getEventTime(),
|
||||
earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
|
||||
|
||||
// Convert these event times to billing times
|
||||
final ImmutableSet<DateTime> billingTimes =
|
||||
getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
|
||||
|
||||
VKey<DomainBase> domainKey =
|
||||
VKey.create(
|
||||
DomainBase.class, recurring.getDomainRepoId(), recurring.getParentKey().getParent());
|
||||
Iterable<OneTime> oneTimesForDomain;
|
||||
if (tm().isOfy()) {
|
||||
oneTimesForDomain = auditedOfy().load().type(OneTime.class).ancestor(domainKey.getOfyKey());
|
||||
} else {
|
||||
oneTimesForDomain =
|
||||
tm().createQueryComposer(OneTime.class)
|
||||
.where("domainRepoId", EQ, recurring.getDomainRepoId())
|
||||
.list();
|
||||
}
|
||||
|
||||
// Determine the billing times that already have OneTime events persisted.
|
||||
ImmutableSet<DateTime> existingBillingTimes =
|
||||
getExistingBillingTimes(oneTimesForDomain, recurring);
|
||||
|
||||
ImmutableSet.Builder<DomainHistory> historyEntriesBuilder = new ImmutableSet.Builder<>();
|
||||
// Create synthetic OneTime events for all billing times that do not yet have
|
||||
// an event persisted.
|
||||
for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
|
||||
// Construct a new HistoryEntry that parents over the OneTime
|
||||
DomainHistory historyEntry =
|
||||
new DomainHistory.Builder()
|
||||
.setBySuperuser(false)
|
||||
.setClientId(recurring.getClientId())
|
||||
.setModificationTime(tm().getTransactionTime())
|
||||
.setDomain(tm().loadByKey(domainKey))
|
||||
.setPeriod(Period.create(1, YEARS))
|
||||
.setReason("Domain autorenewal by ExpandRecurringBillingEventsAction")
|
||||
.setRequestedByRegistrar(false)
|
||||
.setType(DOMAIN_AUTORENEW)
|
||||
// Don't write a domain transaction record if the recurrence was
|
||||
// ended prior to the billing time (i.e. a domain was deleted
|
||||
// during the autorenew grace period).
|
||||
.setDomainTransactionRecords(
|
||||
recurring.getRecurrenceEndTime().isBefore(billingTime)
|
||||
? ImmutableSet.of()
|
||||
: ImmutableSet.of(
|
||||
DomainTransactionRecord.create(
|
||||
tld.getTldStr(),
|
||||
// We report this when the autorenew grace period
|
||||
// ends
|
||||
billingTime,
|
||||
TransactionReportField.netRenewsFieldFromYears(1),
|
||||
1)))
|
||||
.build();
|
||||
historyEntriesBuilder.add(historyEntry);
|
||||
|
||||
DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
|
||||
// Determine the cost for a one-year renewal.
|
||||
Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
|
||||
syntheticOneTimesBuilder.add(
|
||||
new OneTime.Builder()
|
||||
.setBillingTime(billingTime)
|
||||
.setClientId(recurring.getClientId())
|
||||
.setCost(renewCost)
|
||||
.setEventTime(eventTime)
|
||||
.setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
|
||||
.setParent(historyEntry)
|
||||
.setPeriodYears(1)
|
||||
.setReason(recurring.getReason())
|
||||
.setSyntheticCreationTime(executeTime)
|
||||
.setCancellationMatchingBillingEvent(recurring.createVKey())
|
||||
.setTargetId(recurring.getTargetId())
|
||||
.build());
|
||||
}
|
||||
Set<DomainHistory> historyEntries = historyEntriesBuilder.build();
|
||||
Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
|
||||
if (!isDryRun) {
|
||||
ImmutableSet<ImmutableObject> entitiesToSave =
|
||||
new ImmutableSet.Builder<ImmutableObject>()
|
||||
.addAll(historyEntries)
|
||||
.addAll(syntheticOneTimes)
|
||||
.build();
|
||||
tm().putAll(entitiesToSave);
|
||||
}
|
||||
return syntheticOneTimes.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters a set of {@link DateTime}s down to event times that are in scope for a particular
|
||||
* mapreduce run, given the cursor time and the mapreduce execution time.
|
||||
*/
|
||||
protected static ImmutableSet<DateTime> getBillingTimesInScope(
|
||||
Iterable<DateTime> eventTimes,
|
||||
DateTime cursorTime,
|
||||
DateTime executeTime,
|
||||
final Registry tld) {
|
||||
return Streams.stream(eventTimes)
|
||||
.map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
|
||||
.filter(Range.closedOpen(cursorTime, executeTime))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted for a
|
||||
* given recurring billing event.
|
||||
*/
|
||||
private static ImmutableSet<DateTime> getExistingBillingTimes(
|
||||
Iterable<BillingEvent.OneTime> oneTimesForDomain,
|
||||
final BillingEvent.Recurring recurringEvent) {
|
||||
return Streams.stream(oneTimesForDomain)
|
||||
.filter(
|
||||
billingEvent ->
|
||||
recurringEvent
|
||||
.createVKey()
|
||||
.equals(billingEvent.getCancellationMatchingBillingEvent()))
|
||||
.map(OneTime::getBillingTime)
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
|
||||
import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.getLinkedDomainKeys;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
import static google.registry.model.EppResourceUtils.isDeleted;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
@@ -69,6 +70,7 @@ import java.util.logging.Level;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
@@ -86,6 +88,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
||||
@Inject Clock clock;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject @Named(QUEUE_ASYNC_HOST_RENAME) Queue pullQueue;
|
||||
|
||||
@Inject DnsQueue dnsQueue;
|
||||
@Inject RequestStatusChecker requestStatusChecker;
|
||||
@Inject Response response;
|
||||
@Inject Retrier retrier;
|
||||
@@ -153,7 +157,39 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
|
||||
} else {
|
||||
logger.atInfo().log(
|
||||
"Processing asynchronous DNS refresh for renamed hosts: %s", hostKeys.build());
|
||||
runMapreduce(refreshRequests, lock);
|
||||
if (tm().isOfy()) {
|
||||
runMapreduce(refreshRequests, lock);
|
||||
} else {
|
||||
try {
|
||||
refreshRequests.stream()
|
||||
.flatMap(
|
||||
request ->
|
||||
getLinkedDomainKeys(request.hostKey(), request.lastUpdateTime(), null)
|
||||
.stream())
|
||||
.distinct()
|
||||
.map(domainKey -> tm().transact(() -> tm().loadByKey(domainKey).getDomainName()))
|
||||
.forEach(
|
||||
domainName -> {
|
||||
retrier.callWithRetry(
|
||||
() -> dnsQueue.addDomainRefreshTask(domainName),
|
||||
TransientFailureException.class);
|
||||
logger.atInfo().log("Enqueued DNS refresh for domain %s.", domainName);
|
||||
});
|
||||
deleteTasksWithRetry(
|
||||
refreshRequests,
|
||||
getQueue(QUEUE_ASYNC_HOST_RENAME),
|
||||
asyncTaskMetrics,
|
||||
retrier,
|
||||
OperationResult.SUCCESS);
|
||||
} catch (Throwable t) {
|
||||
String message = "Error refreshing DNS on host rename.";
|
||||
logger.atSevere().withCause(t).log(message);
|
||||
response.setPayload(message);
|
||||
response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR);
|
||||
} finally {
|
||||
lock.get().release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
package google.registry.batch;
|
||||
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_FAST;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
@@ -54,6 +54,8 @@ import javax.inject.Inject;
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/resaveAllEppResources",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
// No longer needed in SQL. Subject to future removal.
|
||||
@Deprecated
|
||||
public class ResaveAllEppResourcesAction implements Runnable {
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@@ -104,13 +106,13 @@ public class ResaveAllEppResourcesAction implements Runnable {
|
||||
boolean resaved =
|
||||
tm().transact(
|
||||
() -> {
|
||||
EppResource originalResource = ofy().load().key(resourceKey).now();
|
||||
EppResource originalResource = auditedOfy().load().key(resourceKey).now();
|
||||
EppResource projectedResource =
|
||||
originalResource.cloneProjectedAtTime(tm().getTransactionTime());
|
||||
if (isFast && originalResource.equals(projectedResource)) {
|
||||
return false;
|
||||
} else {
|
||||
ofy().save().entity(projectedResource).now();
|
||||
auditedOfy().save().entity(projectedResource).now();
|
||||
return true;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -21,9 +21,10 @@ import com.google.auto.value.AutoValue;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Streams;
|
||||
import google.registry.backup.AppEngineEnvironment;
|
||||
import google.registry.beam.common.RegistryQuery.QueryComposerFactory;
|
||||
import google.registry.beam.common.RegistryQuery.RegistryQueryFactory;
|
||||
import google.registry.model.ofy.ObjectifyService;
|
||||
import google.registry.persistence.transaction.JpaTransactionManager;
|
||||
import google.registry.persistence.transaction.QueryComposer;
|
||||
import google.registry.persistence.transaction.TransactionManagerFactory;
|
||||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
@@ -68,21 +69,23 @@ public final class RegistryJpaIO {
|
||||
return Read.<R, T>builder().queryFactory(queryFactory).resultMapper(resultMapper).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link Read} connector based on the given {@code jpql} query string.
|
||||
*
|
||||
* <p>User should take care to prevent sql-injection attacks.
|
||||
*/
|
||||
public static <R, T> Read<R, T> read(String jpql, SerializableFunction<R, T> resultMapper) {
|
||||
return Read.<R, T>builder().jpqlQueryFactory(jpql).resultMapper(resultMapper).build();
|
||||
}
|
||||
|
||||
public static <T> Write<T> write() {
|
||||
return Write.<T>builder().build();
|
||||
}
|
||||
|
||||
// TODO(mmuller): Consider detached JpaQueryComposer that works with any JpaTransactionManager
|
||||
// instance, i.e., change composer.buildQuery() to composer.buildQuery(JpaTransactionManager).
|
||||
// This way QueryComposer becomes reusable and serializable (at least with Hibernate), and this
|
||||
// interface would no longer be necessary.
|
||||
public interface QueryComposerFactory<T>
|
||||
extends SerializableFunction<JpaTransactionManager, QueryComposer<T>> {}
|
||||
|
||||
/**
|
||||
* A {@link PTransform transform} that executes a JPA {@link CriteriaQuery} and adds the results
|
||||
* to the BEAM pipeline. Users have the option to transform the results before sending them to the
|
||||
* next stages.
|
||||
* A {@link PTransform transform} that transactionally executes a JPA {@link CriteriaQuery} and
|
||||
* adds the results to the BEAM pipeline. Users have the option to transform the results before
|
||||
* sending them to the next stages.
|
||||
*/
|
||||
@AutoValue
|
||||
public abstract static class Read<R, T> extends PTransform<PBegin, PCollection<T>> {
|
||||
@@ -91,12 +94,10 @@ public final class RegistryJpaIO {
|
||||
|
||||
abstract String name();
|
||||
|
||||
abstract RegistryJpaIO.QueryComposerFactory<R> queryFactory();
|
||||
abstract RegistryQueryFactory<R> queryFactory();
|
||||
|
||||
abstract SerializableFunction<R, T> resultMapper();
|
||||
|
||||
abstract TransactionMode transactionMode();
|
||||
|
||||
abstract Coder<T> coder();
|
||||
|
||||
abstract Builder<R, T> toBuilder();
|
||||
@@ -121,10 +122,6 @@ public final class RegistryJpaIO {
|
||||
return toBuilder().resultMapper(mapper).build();
|
||||
}
|
||||
|
||||
public Read<R, T> withTransactionMode(TransactionMode transactionMode) {
|
||||
return toBuilder().transactionMode(transactionMode).build();
|
||||
}
|
||||
|
||||
public Read<R, T> withCoder(Coder<T> coder) {
|
||||
return toBuilder().coder(coder).build();
|
||||
}
|
||||
@@ -133,7 +130,6 @@ public final class RegistryJpaIO {
|
||||
return new AutoValue_RegistryJpaIO_Read.Builder()
|
||||
.name(DEFAULT_NAME)
|
||||
.resultMapper(x -> x)
|
||||
.transactionMode(TransactionMode.TRANSACTIONAL)
|
||||
.coder(SerializableCoder.of(Serializable.class));
|
||||
}
|
||||
|
||||
@@ -142,23 +138,29 @@ public final class RegistryJpaIO {
|
||||
|
||||
abstract Builder<R, T> name(String name);
|
||||
|
||||
abstract Builder<R, T> queryFactory(RegistryJpaIO.QueryComposerFactory<R> queryFactory);
|
||||
abstract Builder<R, T> queryFactory(RegistryQueryFactory<R> queryFactory);
|
||||
|
||||
abstract Builder<R, T> resultMapper(SerializableFunction<R, T> mapper);
|
||||
|
||||
abstract Builder<R, T> transactionMode(TransactionMode transactionMode);
|
||||
|
||||
abstract Builder<R, T> coder(Coder coder);
|
||||
|
||||
abstract Read<R, T> build();
|
||||
|
||||
Builder<R, T> queryFactory(QueryComposerFactory<R> queryFactory) {
|
||||
return queryFactory(RegistryQuery.createQueryFactory(queryFactory));
|
||||
}
|
||||
|
||||
Builder<R, T> jpqlQueryFactory(String jpql) {
|
||||
return queryFactory(RegistryQuery.createQueryFactory(jpql));
|
||||
}
|
||||
}
|
||||
|
||||
static class QueryRunner<R, T> extends DoFn<Void, T> {
|
||||
private final QueryComposerFactory<R> querySupplier;
|
||||
private final RegistryQueryFactory<R> queryFactory;
|
||||
private final SerializableFunction<R, T> resultMapper;
|
||||
|
||||
QueryRunner(QueryComposerFactory<R> querySupplier, SerializableFunction<R, T> resultMapper) {
|
||||
this.querySupplier = querySupplier;
|
||||
QueryRunner(RegistryQueryFactory<R> queryFactory, SerializableFunction<R, T> resultMapper) {
|
||||
this.queryFactory = queryFactory;
|
||||
this.resultMapper = resultMapper;
|
||||
}
|
||||
|
||||
@@ -171,17 +173,12 @@ public final class RegistryJpaIO {
|
||||
jpaTm()
|
||||
.transactNoRetry(
|
||||
() ->
|
||||
querySupplier.apply(jpaTm()).stream()
|
||||
queryFactory.apply(jpaTm()).stream()
|
||||
.map(resultMapper::apply)
|
||||
.forEach(outputReceiver::output));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum TransactionMode {
|
||||
NOT_TRANSACTIONAL,
|
||||
TRANSACTIONAL;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.common;
|
||||
|
||||
import google.registry.persistence.transaction.JpaTransactionManager;
|
||||
import google.registry.persistence.transaction.QueryComposer;
|
||||
import java.util.stream.Stream;
|
||||
import javax.persistence.EntityManager;
|
||||
import javax.persistence.Query;
|
||||
import org.apache.beam.sdk.transforms.SerializableFunction;
|
||||
|
||||
/** Interface for query instances used by {@link RegistryJpaIO.Read}. */
|
||||
public interface RegistryQuery<T> {
|
||||
Stream<T> stream();
|
||||
|
||||
/** Factory for {@link RegistryQuery}. */
|
||||
interface RegistryQueryFactory<T>
|
||||
extends SerializableFunction<JpaTransactionManager, RegistryQuery<T>> {}
|
||||
|
||||
// TODO(mmuller): Consider detached JpaQueryComposer that works with any JpaTransactionManager
|
||||
// instance, i.e., change composer.buildQuery() to composer.buildQuery(JpaTransactionManager).
|
||||
// This way QueryComposer becomes reusable and serializable (at least with Hibernate), and this
|
||||
// interface would no longer be necessary.
|
||||
interface QueryComposerFactory<T>
|
||||
extends SerializableFunction<JpaTransactionManager, QueryComposer<T>> {}
|
||||
|
||||
/**
|
||||
* Returns a {@link RegistryQueryFactory} that creates a JPQL query from constant text.
|
||||
*
|
||||
* @param <T> Type of each row in the result set, {@link Object} in single-select queries, and
|
||||
* {@code Object[]} in multi-select queries.
|
||||
*/
|
||||
@SuppressWarnings("unchecked") // query.getResultStream: jpa api uses raw type
|
||||
static <T> RegistryQueryFactory<T> createQueryFactory(String jpql) {
|
||||
return (JpaTransactionManager jpa) ->
|
||||
() -> {
|
||||
EntityManager entityManager = jpa.getEntityManager();
|
||||
Query query = entityManager.createQuery(jpql);
|
||||
return query.getResultStream().map(e -> detach(entityManager, e));
|
||||
};
|
||||
}
|
||||
|
||||
static <T> RegistryQueryFactory<T> createQueryFactory(
|
||||
QueryComposerFactory<T> queryComposerFactory) {
|
||||
return (JpaTransactionManager jpa) ->
|
||||
() -> queryComposerFactory.apply(jpa).withAutoDetachOnLoad(true).stream();
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes an object from the JPA session cache if applicable.
|
||||
*
|
||||
* @param object An object that represents a row in the result set. It may be a JPA entity, a
|
||||
* non-entity object, or an array that holds JPA entities and/or non-entities.
|
||||
*/
|
||||
static <T> T detach(EntityManager entityManager, T object) {
|
||||
if (object.getClass().isArray()) {
|
||||
for (Object arrayElement : (Object[]) object) {
|
||||
detachObject(entityManager, arrayElement);
|
||||
}
|
||||
} else {
|
||||
detachObject(entityManager, object);
|
||||
}
|
||||
return object;
|
||||
}
|
||||
|
||||
static void detachObject(EntityManager entityManager, Object object) {
|
||||
Class<?> objectClass = object.getClass();
|
||||
if (objectClass.isPrimitive() || objectClass == String.class) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
entityManager.detach(object);
|
||||
} catch (IllegalArgumentException e) {
|
||||
// Not an entity. Do nothing.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -251,7 +251,14 @@ public abstract class BillingEvent implements Serializable {
|
||||
InvoiceGroupingKey getInvoiceGroupingKey() {
|
||||
return new AutoValue_BillingEvent_InvoiceGroupingKey(
|
||||
billingTime().toLocalDate().withDayOfMonth(1).toString(),
|
||||
billingTime().toLocalDate().withDayOfMonth(1).plusYears(years()).minusDays(1).toString(),
|
||||
years() == 0
|
||||
? ""
|
||||
: billingTime()
|
||||
.toLocalDate()
|
||||
.withDayOfMonth(1)
|
||||
.plusYears(years())
|
||||
.minusDays(1)
|
||||
.toString(),
|
||||
billingId(),
|
||||
String.format("%s - %s", registrarId(), tld()),
|
||||
String.format("%s | TLD: %s | TERM: %d-year", action(), tld(), years()),
|
||||
|
||||
@@ -23,8 +23,10 @@ import dagger.Component;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.beam.common.RegistryJpaIO.Read;
|
||||
import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
|
||||
import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch;
|
||||
import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
|
||||
import google.registry.util.Retrier;
|
||||
@@ -97,20 +99,9 @@ public class Spec11Pipeline implements Serializable {
|
||||
|
||||
void setupPipeline(Pipeline pipeline) {
|
||||
PCollection<Subdomain> domains =
|
||||
pipeline.apply(
|
||||
"Read active domains from BigQuery",
|
||||
BigQueryIO.read(Subdomain::parseFromRecord)
|
||||
.fromQuery(
|
||||
SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
|
||||
.put("PROJECT_ID", options.getProject())
|
||||
.put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("DOMAIN_BASE_TABLE", "DomainBase")
|
||||
.build())
|
||||
.withCoder(SerializableCoder.of(Subdomain.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
options.getDatabase().equals("DATASTORE")
|
||||
? readFromBigQuery(options, pipeline)
|
||||
: readFromCloudSql(pipeline);
|
||||
|
||||
PCollection<KV<Subdomain, ThreatMatch>> threatMatches =
|
||||
domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn));
|
||||
@@ -119,6 +110,47 @@ public class Spec11Pipeline implements Serializable {
|
||||
saveToGcs(threatMatches, options);
|
||||
}
|
||||
|
||||
static PCollection<Subdomain> readFromCloudSql(Pipeline pipeline) {
|
||||
Read<Object[], Subdomain> read =
|
||||
RegistryJpaIO.read(
|
||||
"select d, r.emailAddress from Domain d join Registrar r on"
|
||||
+ " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
|
||||
+ " and d.deletionTime > now()",
|
||||
Spec11Pipeline::parseRow);
|
||||
|
||||
return pipeline.apply("Read active domains from Cloud SQL", read);
|
||||
}
|
||||
|
||||
static PCollection<Subdomain> readFromBigQuery(Spec11PipelineOptions options, Pipeline pipeline) {
|
||||
return pipeline.apply(
|
||||
"Read active domains from BigQuery",
|
||||
BigQueryIO.read(Subdomain::parseFromRecord)
|
||||
.fromQuery(
|
||||
SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
|
||||
.put("PROJECT_ID", options.getProject())
|
||||
.put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
|
||||
.put("REGISTRAR_TABLE", "Registrar")
|
||||
.put("DOMAIN_BASE_TABLE", "DomainBase")
|
||||
.build())
|
||||
.withCoder(SerializableCoder.of(Subdomain.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
}
|
||||
|
||||
private static Subdomain parseRow(Object[] row) {
|
||||
DomainBase domainBase = (DomainBase) row[0];
|
||||
String emailAddress = (String) row[1];
|
||||
if (emailAddress == null) {
|
||||
emailAddress = "";
|
||||
}
|
||||
return Subdomain.create(
|
||||
domainBase.getDomainName(),
|
||||
domainBase.getRepoId(),
|
||||
domainBase.getCurrentSponsorClientId(),
|
||||
emailAddress);
|
||||
}
|
||||
|
||||
static void saveToSql(
|
||||
PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
|
||||
String transformId = "Spec11 Threat Matches";
|
||||
|
||||
@@ -34,4 +34,9 @@ public interface Spec11PipelineOptions extends RegistryPipelineOptions {
|
||||
String getReportingBucketUrl();
|
||||
|
||||
void setReportingBucketUrl(String value);
|
||||
|
||||
@Description("The database to read data from.")
|
||||
String getDatabase();
|
||||
|
||||
void setDatabase(String value);
|
||||
}
|
||||
|
||||
@@ -203,7 +203,7 @@ public final class RegistryConfig {
|
||||
* Configuration for analytics services installed in the web console.
|
||||
*
|
||||
* @see google.registry.ui.server.registrar.ConsoleUiAction
|
||||
* @see google.registry.ui.soy.AnalyticsSoyInfo
|
||||
* @see google.registry.ui.soy.registrar.AnalyticsSoyInfo
|
||||
*/
|
||||
@Provides
|
||||
@Config("analyticsConfig")
|
||||
@@ -1502,22 +1502,6 @@ public final class RegistryConfig {
|
||||
CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether or not to replay commit logs to the SQL database after export to GCS.
|
||||
*
|
||||
* <p>If true, we will trigger the {@link google.registry.backup.ReplayCommitLogsToSqlAction}
|
||||
* after the {@link google.registry.backup.ExportCommitLogDiffAction} to load the commit logs and
|
||||
* replay them to SQL.
|
||||
*/
|
||||
public static boolean getCloudSqlReplayCommitLogs() {
|
||||
return CONFIG_SETTINGS.get().cloudSql.replayCommitLogs;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void overrideCloudSqlReplayCommitLogs(boolean replayCommitLogs) {
|
||||
CONFIG_SETTINGS.get().cloudSql.replayCommitLogs = replayCommitLogs;
|
||||
}
|
||||
|
||||
/** Returns the roid suffix to be used for the roids of all contacts and hosts. */
|
||||
public static String getContactAndHostRoidSuffix() {
|
||||
return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;
|
||||
|
||||
@@ -127,7 +127,6 @@ public class RegistryConfigSettings {
|
||||
public String username;
|
||||
public String instanceConnectionName;
|
||||
public boolean replicateTransactions;
|
||||
public boolean replayCommitLogs;
|
||||
}
|
||||
|
||||
/** Configuration for Apache Beam (Cloud Dataflow). */
|
||||
|
||||
@@ -230,8 +230,6 @@ cloudSql:
|
||||
# Set this to true to replicate cloud SQL transactions to datastore in the
|
||||
# background.
|
||||
replicateTransactions: false
|
||||
# Set this to true to enable replay of commit logs to SQL
|
||||
replayCommitLogs: false
|
||||
|
||||
cloudDns:
|
||||
# Set both properties to null in Production.
|
||||
|
||||
@@ -199,6 +199,15 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
|
||||
<description>
|
||||
|
||||
@@ -237,6 +237,12 @@
|
||||
<url-pattern>/_dr/task/killCommitLogs</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Replays Datastore commit logs to SQL. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/replayCommitLogsToSql</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- MapReduce servlet. -->
|
||||
<servlet>
|
||||
<servlet-name>mapreduce</servlet-name>
|
||||
@@ -391,6 +397,12 @@
|
||||
<url-pattern>/_dr/task/wipeOutDatastore</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Action to create synthetic history entries during async replication to SQL -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/createSyntheticHistoryEntries</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Security config -->
|
||||
<security-constraint>
|
||||
<web-resource-collection>
|
||||
|
||||
@@ -41,6 +41,11 @@
|
||||
<property name="nsHosts" direction="asc"/>
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For deleting expired not-previously-deleted domains. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="deletionTime" direction="asc"/>
|
||||
<property name="autorenewEndTime" direction="asc"/>
|
||||
</datastore-index>
|
||||
<!-- For RDAP searches by linked nameserver. -->
|
||||
<datastore-index kind="DomainBase" ancestor="false" source="manual">
|
||||
<property name="nsHosts" direction="asc"/>
|
||||
|
||||
@@ -200,4 +200,12 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
</cronentries>
|
||||
|
||||
@@ -1,6 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<cronentries>
|
||||
|
||||
<cron>
|
||||
<url>/_dr/task/rdeStaging</url>
|
||||
<description>
|
||||
This job generates a full RDE escrow deposit as a single gigantic XML document
|
||||
and streams it to cloud storage. When this job has finished successfully, it'll
|
||||
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
|
||||
</description>
|
||||
<schedule>every day 00:07</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
|
||||
<description>
|
||||
@@ -100,4 +111,21 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
|
||||
<description>
|
||||
This job checkpoints the commit log buckets and exports the diff since last checkpoint to GCS.
|
||||
</description>
|
||||
<schedule>every 3 minutes synchronized</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
</cronentries>
|
||||
|
||||
@@ -229,4 +229,12 @@
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
|
||||
<description>
|
||||
Replays recent commit logs from Datastore to the SQL secondary backend.
|
||||
</description>
|
||||
<schedule>every 3 minutes</schedule>
|
||||
<target>backend</target>
|
||||
</cron>
|
||||
</cronentries>
|
||||
|
||||
@@ -19,9 +19,10 @@ import static com.google.common.base.Verify.verifyNotNull;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
import static google.registry.model.registry.Registries.getTldsOfType;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
@@ -45,12 +46,14 @@ import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.List;
|
||||
import java.util.function.Supplier;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -70,9 +73,13 @@ public class ExportDomainListsAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final int MAX_NUM_REDUCE_SHARDS = 100;
|
||||
public static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject DriveConnection driveConnection;
|
||||
|
||||
@Inject @Config("domainListsGcsBucket") String gcsBucket;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject ExportDomainListsAction() {}
|
||||
@@ -81,15 +88,99 @@ public class ExportDomainListsAction implements Runnable {
|
||||
public void run() {
|
||||
ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
|
||||
logger.atInfo().log("Exporting domain lists for tlds %s", realTlds);
|
||||
mrRunner
|
||||
.setJobName("Export domain lists")
|
||||
.setModuleName("backend")
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(DateTime.now(UTC), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Export domain lists")
|
||||
.setModuleName("backend")
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(clock.nowUtc(), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
realTlds.forEach(
|
||||
tld -> {
|
||||
List<String> domains =
|
||||
tm().transact(
|
||||
() ->
|
||||
// Note that if we had "creationTime <= :now" in the condition (not
|
||||
// necessary as there is no pending creation, the order of deletionTime
|
||||
// and creationTime in the query would have been significant and it
|
||||
// should come after deletionTime. When Hibernate substitutes "now" it
|
||||
// will first validate that the **first** field that is to be compared
|
||||
// with it (deletionTime) is assignable from the substituted Java object
|
||||
// (click.nowUtc()). Since creationTime is a CreateAutoTimestamp, if it
|
||||
// comes first, we will need to substitute "now" with
|
||||
// CreateAutoTimestamp.create(clock.nowUtc()). This might look a bit
|
||||
// strange as the Java object type is clearly incompatible between the
|
||||
// two fields deletionTime (DateTime) and creationTime, yet they are
|
||||
// compared with the same "now". It is actually OK because in the end
|
||||
// Hibernate converts everything to SQL types (and Java field names to
|
||||
// SQL column names) to run the query. Both CreateAutoTimestamp and
|
||||
// DateTime are persisted as timestamp_z in SQL. It is only the
|
||||
// validation that compares the Java types, and only with the first
|
||||
// field that compares with the substituted value.
|
||||
jpaTm()
|
||||
.query(
|
||||
"SELECT fullyQualifiedDomainName FROM Domain "
|
||||
+ "WHERE tld = :tld "
|
||||
+ "AND deletionTime > :now "
|
||||
+ "ORDER by fullyQualifiedDomainName ASC",
|
||||
String.class)
|
||||
.setParameter("tld", tld)
|
||||
.setParameter("now", clock.nowUtc())
|
||||
.getResultList());
|
||||
String domainsList = Joiner.on("\n").join(domains);
|
||||
logger.atInfo().log(
|
||||
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize);
|
||||
exportToDrive(tld, domainsList, driveConnection);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
protected static boolean exportToDrive(
|
||||
String tld, String domains, DriveConnection driveConnection) {
|
||||
verifyNotNull(driveConnection, "Expecting non-null driveConnection");
|
||||
try {
|
||||
Registry registry = Registry.get(tld);
|
||||
if (registry.getDriveFolderId() == null) {
|
||||
logger.atInfo().log(
|
||||
"Skipping registered domains export for TLD %s because Drive folder isn't specified",
|
||||
tld);
|
||||
} else {
|
||||
String resultMsg =
|
||||
driveConnection.createOrUpdateFile(
|
||||
REGISTERED_DOMAINS_FILENAME,
|
||||
MediaType.PLAIN_TEXT_UTF_8,
|
||||
registry.getDriveFolderId(),
|
||||
domains.getBytes(UTF_8));
|
||||
logger.atInfo().log(
|
||||
"Exporting registered domains succeeded for TLD %s, response was: %s", tld, resultMsg);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to Drive, skipping...", tld);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
protected static boolean exportToGcs(
|
||||
String tld, String domains, String gcsBucket, int gcsBufferSize) {
|
||||
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to GCS, skipping...", tld);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static class ExportDomainListsMapper extends Mapper<DomainBase, String, String> {
|
||||
@@ -122,9 +213,6 @@ public class ExportDomainListsAction implements Runnable {
|
||||
private static Supplier<DriveConnection> driveConnectionSupplier =
|
||||
Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection());
|
||||
|
||||
static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
|
||||
static final MediaType EXPORT_MIME_TYPE = MediaType.PLAIN_TEXT_UTF_8;
|
||||
|
||||
private final String gcsBucket;
|
||||
private final int gcsBufferSize;
|
||||
|
||||
@@ -147,53 +235,21 @@ public class ExportDomainListsAction implements Runnable {
|
||||
driveConnection = driveConnectionSupplier.get();
|
||||
}
|
||||
|
||||
private void exportToDrive(String tld, String domains) {
|
||||
verifyNotNull(driveConnection, "expecting non-null driveConnection");
|
||||
try {
|
||||
Registry registry = Registry.get(tld);
|
||||
if (registry.getDriveFolderId() == null) {
|
||||
logger.atInfo().log(
|
||||
"Skipping registered domains export for TLD %s because Drive folder isn't specified",
|
||||
tld);
|
||||
} else {
|
||||
String resultMsg =
|
||||
driveConnection.createOrUpdateFile(
|
||||
REGISTERED_DOMAINS_FILENAME,
|
||||
EXPORT_MIME_TYPE,
|
||||
registry.getDriveFolderId(),
|
||||
domains.getBytes(UTF_8));
|
||||
logger.atInfo().log(
|
||||
"Exporting registered domains succeeded for TLD %s, response was: %s",
|
||||
tld, resultMsg);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to Drive", tld);
|
||||
}
|
||||
getContext().incrementCounter("domain lists written out to Drive");
|
||||
}
|
||||
|
||||
private void exportToGcs(String tld, String domains) {
|
||||
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (IOException e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to GCS.", tld);
|
||||
}
|
||||
getContext().incrementCounter("domain lists written out to GCS");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reduce(String tld, ReducerInput<String> fqdns) {
|
||||
ImmutableList<String> domains = ImmutableList.sortedCopyOf(() -> fqdns);
|
||||
String domainsList = Joiner.on('\n').join(domains);
|
||||
logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList);
|
||||
exportToDrive(tld, domainsList);
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize)) {
|
||||
getContext().incrementCounter("domain lists successful written out to GCS");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to GCS");
|
||||
}
|
||||
if (exportToDrive(tld, domainsList, driveConnection)) {
|
||||
getContext().incrementCounter("domain lists successfully written out to Drive");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to Drive");
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
|
||||
@@ -32,12 +32,12 @@ import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.label.PremiumList.PremiumListEntry;
|
||||
import google.registry.model.registry.label.PremiumListDualDao;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.RequestParameters;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.schema.tld.PremiumListDao;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
@@ -113,7 +113,7 @@ public class ExportPremiumTermsAction implements Runnable {
|
||||
"Skipping premium terms export for TLD %s because Drive folder isn't specified", tld);
|
||||
return Optional.of("Skipping export because no Drive folder is associated with this TLD");
|
||||
}
|
||||
if (registry.getPremiumList() == null) {
|
||||
if (!registry.getPremiumList().isPresent()) {
|
||||
logger.atInfo().log("No premium terms to export for TLD %s", tld);
|
||||
return Optional.of("No premium lists configured");
|
||||
}
|
||||
@@ -137,11 +137,13 @@ public class ExportPremiumTermsAction implements Runnable {
|
||||
}
|
||||
|
||||
private String getFormattedPremiumTerms(Registry registry) {
|
||||
String premiumListName = registry.getPremiumList().getName();
|
||||
checkState(registry.getPremiumList().isPresent(), "%s does not have a premium list", tld);
|
||||
String premiumListName = registry.getPremiumList().get().getName();
|
||||
checkState(
|
||||
PremiumListDualDao.exists(premiumListName), "Could not load premium list for " + tld);
|
||||
PremiumListDao.getLatestRevision(premiumListName).isPresent(),
|
||||
"Could not load premium list for " + tld);
|
||||
SortedSet<String> premiumTerms =
|
||||
Streams.stream(PremiumListDualDao.loadAllPremiumListEntries(premiumListName))
|
||||
Streams.stream(PremiumListDao.loadAllPremiumListEntries(premiumListName))
|
||||
.map(PremiumListEntry::toString)
|
||||
.collect(ImmutableSortedSet.toImmutableSortedSet(String::compareTo));
|
||||
|
||||
|
||||
@@ -213,50 +213,78 @@ public class FlowModule {
|
||||
return Strings.nullToEmpty(((Poll) eppInput.getCommandWrapper().getCommand()).getMessageId());
|
||||
}
|
||||
|
||||
private static <B extends HistoryEntry.Builder<? extends HistoryEntry, ?>>
|
||||
B makeHistoryEntryBuilder(
|
||||
B builder,
|
||||
Trid trid,
|
||||
byte[] inputXmlBytes,
|
||||
boolean isSuperuser,
|
||||
String clientId,
|
||||
EppInput eppInput) {
|
||||
builder
|
||||
.setTrid(trid)
|
||||
.setXmlBytes(inputXmlBytes)
|
||||
.setBySuperuser(isSuperuser)
|
||||
.setClientId(clientId);
|
||||
Optional<MetadataExtension> metadataExtension =
|
||||
eppInput.getSingleExtension(MetadataExtension.class);
|
||||
metadataExtension.ifPresent(
|
||||
extension ->
|
||||
builder
|
||||
.setReason(extension.getReason())
|
||||
.setRequestedByRegistrar(extension.getRequestedByRegistrar()));
|
||||
return builder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a partially filled in {@link HistoryEntry} builder.
|
||||
* Provides a partially filled in {@link ContactHistory.Builder}
|
||||
*
|
||||
* <p>This is not marked with {@link FlowScope} so that each retry gets a fresh one. Otherwise,
|
||||
* the fact that the builder is one-use would cause NPEs.
|
||||
*/
|
||||
@Provides
|
||||
static HistoryEntry.Builder provideHistoryEntryBuilder(
|
||||
static ContactHistory.Builder provideContactHistoryBuilder(
|
||||
Trid trid,
|
||||
@InputXml byte[] inputXmlBytes,
|
||||
@Superuser boolean isSuperuser,
|
||||
@ClientId String clientId,
|
||||
EppInput eppInput) {
|
||||
HistoryEntry.Builder historyBuilder =
|
||||
new HistoryEntry.Builder()
|
||||
.setTrid(trid)
|
||||
.setXmlBytes(inputXmlBytes)
|
||||
.setBySuperuser(isSuperuser)
|
||||
.setClientId(clientId);
|
||||
Optional<MetadataExtension> metadataExtension =
|
||||
eppInput.getSingleExtension(MetadataExtension.class);
|
||||
metadataExtension.ifPresent(
|
||||
extension ->
|
||||
historyBuilder
|
||||
.setReason(extension.getReason())
|
||||
.setRequestedByRegistrar(extension.getRequestedByRegistrar()));
|
||||
return historyBuilder;
|
||||
return makeHistoryEntryBuilder(
|
||||
new ContactHistory.Builder(), trid, inputXmlBytes, isSuperuser, clientId, eppInput);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a partially filled in {@link HostHistory.Builder}
|
||||
*
|
||||
* <p>This is not marked with {@link FlowScope} so that each retry gets a fresh one. Otherwise,
|
||||
* the fact that the builder is one-use would cause NPEs.
|
||||
*/
|
||||
@Provides
|
||||
static ContactHistory.Builder provideContactHistoryBuilder(
|
||||
HistoryEntry.Builder historyEntryBuilder) {
|
||||
return new ContactHistory.Builder().copyFrom(historyEntryBuilder);
|
||||
static HostHistory.Builder provideHostHistoryBuilder(
|
||||
Trid trid,
|
||||
@InputXml byte[] inputXmlBytes,
|
||||
@Superuser boolean isSuperuser,
|
||||
@ClientId String clientId,
|
||||
EppInput eppInput) {
|
||||
return makeHistoryEntryBuilder(
|
||||
new HostHistory.Builder(), trid, inputXmlBytes, isSuperuser, clientId, eppInput);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a partially filled in {@link DomainHistory.Builder}
|
||||
*
|
||||
* <p>This is not marked with {@link FlowScope} so that each retry gets a fresh one. Otherwise,
|
||||
* the fact that the builder is one-use would cause NPEs.
|
||||
*/
|
||||
@Provides
|
||||
static DomainHistory.Builder provideDomainHistoryBuilder(
|
||||
HistoryEntry.Builder historyEntryBuilder) {
|
||||
return new DomainHistory.Builder().copyFrom(historyEntryBuilder);
|
||||
}
|
||||
|
||||
@Provides
|
||||
static HostHistory.Builder provideHostHistoryBuilder(HistoryEntry.Builder historyEntryBuilder) {
|
||||
return new HostHistory.Builder().copyFrom(historyEntryBuilder);
|
||||
Trid trid,
|
||||
@InputXml byte[] inputXmlBytes,
|
||||
@Superuser boolean isSuperuser,
|
||||
@ClientId String clientId,
|
||||
EppInput eppInput) {
|
||||
return makeHistoryEntryBuilder(
|
||||
new DomainHistory.Builder(), trid, inputXmlBytes, isSuperuser, clientId, eppInput);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -45,7 +45,7 @@ import google.registry.model.eppinput.ResourceCommand;
|
||||
import google.registry.model.eppoutput.EppResponse;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.HashSet;
|
||||
import java.util.Optional;
|
||||
@@ -104,8 +104,7 @@ public final class DomainClaimsCheckFlow implements Flow {
|
||||
verifyClaimsPeriodNotEnded(registry, now);
|
||||
}
|
||||
}
|
||||
Optional<String> claimKey =
|
||||
ClaimsListDualDatabaseDao.get().getClaimKey(parsedDomain.parts().get(0));
|
||||
Optional<String> claimKey = ClaimsListDao.get().getClaimKey(parsedDomain.parts().get(0));
|
||||
launchChecksBuilder.add(
|
||||
LaunchCheck.create(
|
||||
LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null)));
|
||||
|
||||
@@ -251,8 +251,10 @@ public final class DomainDeleteFlow implements TransactionalFlow {
|
||||
buildDomainHistory(newDomain, registry, now, durationUntilDelete, inAddGracePeriod);
|
||||
updateForeignKeyIndexDeletionTime(newDomain);
|
||||
handlePendingTransferOnDelete(existingDomain, newDomain, now, domainHistory);
|
||||
// Close the autorenew billing event and poll message. This may delete the poll message.
|
||||
updateAutorenewRecurrenceEndTime(existingDomain, now);
|
||||
// Close the autorenew billing event and poll message. This may delete the poll message. Store
|
||||
// the updated recurring billing event, we'll need it later and can't reload it.
|
||||
BillingEvent.Recurring recurringBillingEvent =
|
||||
updateAutorenewRecurrenceEndTime(existingDomain, now);
|
||||
// If there's a pending transfer, the gaining client's autorenew billing
|
||||
// event and poll message will already have been deleted in
|
||||
// ResourceDeleteFlow since it's listed in serverApproveEntities.
|
||||
@@ -275,7 +277,8 @@ public final class DomainDeleteFlow implements TransactionalFlow {
|
||||
newDomain.getDeletionTime().isAfter(now)
|
||||
? SUCCESS_WITH_ACTION_PENDING
|
||||
: SUCCESS)
|
||||
.setResponseExtensions(getResponseExtensions(existingDomain, now))
|
||||
.setResponseExtensions(
|
||||
getResponseExtensions(recurringBillingEvent, existingDomain, now))
|
||||
.build());
|
||||
persistEntityChanges(entityChanges);
|
||||
return responseBuilder
|
||||
@@ -377,7 +380,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
|
||||
|
||||
@Nullable
|
||||
private ImmutableList<FeeTransformResponseExtension> getResponseExtensions(
|
||||
DomainBase existingDomain, DateTime now) {
|
||||
BillingEvent.Recurring recurringBillingEvent, DomainBase existingDomain, DateTime now) {
|
||||
FeeTransformResponseExtension.Builder feeResponseBuilder = getDeleteResponseBuilder();
|
||||
if (feeResponseBuilder == null) {
|
||||
return ImmutableList.of();
|
||||
@@ -385,7 +388,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
|
||||
ImmutableList.Builder<Credit> creditsBuilder = new ImmutableList.Builder<>();
|
||||
for (GracePeriod gracePeriod : existingDomain.getGracePeriods()) {
|
||||
if (gracePeriod.hasBillingEvent()) {
|
||||
Money cost = getGracePeriodCost(gracePeriod, now);
|
||||
Money cost = getGracePeriodCost(recurringBillingEvent, gracePeriod, now);
|
||||
creditsBuilder.add(Credit.create(
|
||||
cost.negated().getAmount(), FeeType.CREDIT, gracePeriod.getType().getXmlName()));
|
||||
feeResponseBuilder.setCurrency(checkNotNull(cost.getCurrencyUnit()));
|
||||
@@ -398,12 +401,12 @@ public final class DomainDeleteFlow implements TransactionalFlow {
|
||||
return ImmutableList.of(feeResponseBuilder.setCredits(credits).build());
|
||||
}
|
||||
|
||||
private Money getGracePeriodCost(GracePeriod gracePeriod, DateTime now) {
|
||||
private Money getGracePeriodCost(
|
||||
BillingEvent.Recurring recurringBillingEvent, GracePeriod gracePeriod, DateTime now) {
|
||||
if (gracePeriod.getType() == GracePeriodStatus.AUTO_RENEW) {
|
||||
// If we updated the autorenew billing event, reuse it.
|
||||
DateTime autoRenewTime =
|
||||
tm().loadByKey(checkNotNull(gracePeriod.getRecurringBillingEvent()))
|
||||
.getRecurrenceTimeOfYear()
|
||||
.getLastInstanceBeforeOrAt(now);
|
||||
recurringBillingEvent.getRecurrenceTimeOfYear().getLastInstanceBeforeOrAt(now);
|
||||
return getDomainRenewCost(targetId, autoRenewTime, 1);
|
||||
}
|
||||
return tm().loadByKey(checkNotNull(gracePeriod.getOneTimeBillingEvent())).getCost();
|
||||
|
||||
@@ -25,11 +25,8 @@ import static com.google.common.collect.Iterables.any;
|
||||
import static com.google.common.collect.Sets.difference;
|
||||
import static com.google.common.collect.Sets.intersection;
|
||||
import static com.google.common.collect.Sets.union;
|
||||
import static google.registry.model.DatabaseMigrationUtils.getPrimaryDatabase;
|
||||
import static google.registry.model.common.DatabaseTransitionSchedule.PrimaryDatabase.DATASTORE;
|
||||
import static google.registry.model.common.DatabaseTransitionSchedule.TransitionId.REPLAYED_ENTITIES;
|
||||
import static google.registry.model.domain.DomainBase.MAX_REGISTRATION_YEARS;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.model.registry.Registries.findTldForName;
|
||||
import static google.registry.model.registry.Registries.getTlds;
|
||||
import static google.registry.model.registry.Registry.TldState.GENERAL_AVAILABILITY;
|
||||
@@ -129,7 +126,7 @@ import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.reporting.DomainTransactionRecord;
|
||||
import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.tmch.ClaimsListDualDatabaseDao;
|
||||
import google.registry.model.tmch.ClaimsListDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.tldconfig.idn.IdnLabelValidator;
|
||||
import google.registry.util.Idn;
|
||||
@@ -520,8 +517,10 @@ public class DomainFlowUtils {
|
||||
* <p>This may end up deleting the poll message (if closing the message interval) or recreating it
|
||||
* (if opening the message interval). This may cause an autorenew billing event to have an end
|
||||
* time earlier than its event time (i.e. if it's being ended before it was ever triggered).
|
||||
*
|
||||
* <p>Returns the new autorenew recurring billing event.
|
||||
*/
|
||||
public static void updateAutorenewRecurrenceEndTime(DomainBase domain, DateTime newEndTime) {
|
||||
public static Recurring updateAutorenewRecurrenceEndTime(DomainBase domain, DateTime newEndTime) {
|
||||
Optional<PollMessage.Autorenew> autorenewPollMessage =
|
||||
tm().loadByKeyIfPresent(domain.getAutorenewPollMessage());
|
||||
|
||||
@@ -548,8 +547,13 @@ public class DomainFlowUtils {
|
||||
tm().put(updatedAutorenewPollMessage);
|
||||
}
|
||||
|
||||
Recurring recurring = tm().loadByKey(domain.getAutorenewBillingEvent());
|
||||
tm().put(recurring.asBuilder().setRecurrenceEndTime(newEndTime).build());
|
||||
Recurring recurring =
|
||||
tm().loadByKey(domain.getAutorenewBillingEvent())
|
||||
.asBuilder()
|
||||
.setRecurrenceEndTime(newEndTime)
|
||||
.build();
|
||||
tm().put(recurring);
|
||||
return recurring;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -993,8 +997,7 @@ public class DomainFlowUtils {
|
||||
static void verifyClaimsNoticeIfAndOnlyIfNeeded(
|
||||
InternetDomainName domainName, boolean hasSignedMarks, boolean hasClaimsNotice)
|
||||
throws EppException {
|
||||
boolean isInClaimsList =
|
||||
ClaimsListDualDatabaseDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
|
||||
boolean isInClaimsList = ClaimsListDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
|
||||
if (hasClaimsNotice && !isInClaimsList) {
|
||||
throw new UnexpectedClaimsNoticeException(domainName.toString());
|
||||
}
|
||||
@@ -1083,8 +1086,8 @@ public class DomainFlowUtils {
|
||||
|
||||
private static List<? extends HistoryEntry> findRecentHistoryEntries(
|
||||
DomainBase domainBase, DateTime now, Duration maxSearchPeriod) {
|
||||
if (getPrimaryDatabase(REPLAYED_ENTITIES).equals(DATASTORE)) {
|
||||
return ofy()
|
||||
if (tm().isOfy()) {
|
||||
return auditedOfy()
|
||||
.load()
|
||||
.type(HistoryEntry.class)
|
||||
.ancestor(domainBase)
|
||||
|
||||
@@ -290,10 +290,7 @@ public final class DomainUpdateFlow implements TransactionalFlow {
|
||||
|
||||
/** Some status updates cost money. Bill only once no matter how many of them are changed. */
|
||||
private Optional<BillingEvent.OneTime> createBillingEventForStatusUpdates(
|
||||
DomainBase existingDomain,
|
||||
DomainBase newDomain,
|
||||
HistoryEntry historyEntry,
|
||||
DateTime now) {
|
||||
DomainBase existingDomain, DomainBase newDomain, DomainHistory historyEntry, DateTime now) {
|
||||
Optional<MetadataExtension> metadataExtension =
|
||||
eppInput.getSingleExtension(MetadataExtension.class);
|
||||
if (metadataExtension.isPresent() && metadataExtension.get().getRequestedByRegistrar()) {
|
||||
|
||||
@@ -15,57 +15,29 @@
|
||||
package google.registry.flows.poll;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
|
||||
import static google.registry.persistence.transaction.QueryComposer.Comparator.LTE;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
import com.googlecode.objectify.cmd.Query;
|
||||
import google.registry.model.poll.PollMessage;
|
||||
import google.registry.persistence.transaction.QueryComposer;
|
||||
import java.util.Optional;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Static utility functions for poll flows. */
|
||||
public final class PollFlowUtils {
|
||||
|
||||
public static final String SQL_POLL_MESSAGE_QUERY =
|
||||
"FROM PollMessage WHERE clientId = :registrarId AND eventTime <= :now ORDER BY eventTime ASC";
|
||||
private static final String SQL_POLL_MESSAGE_COUNT_QUERY =
|
||||
"SELECT COUNT(*) FROM PollMessage WHERE clientId = :registrarId AND eventTime <= :now";
|
||||
|
||||
/** Returns the number of poll messages for the given registrar that are not in the future. */
|
||||
public static int getPollMessageCount(String registrarId, DateTime now) {
|
||||
if (tm().isOfy()) {
|
||||
return datastorePollMessageQuery(registrarId, now).count();
|
||||
} else {
|
||||
return jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm()
|
||||
.query(SQL_POLL_MESSAGE_COUNT_QUERY, Long.class)
|
||||
.setParameter("registrarId", registrarId)
|
||||
.setParameter("now", now)
|
||||
.getSingleResult()
|
||||
.intValue());
|
||||
}
|
||||
return transactIfJpaTm(() -> createPollMessageQuery(registrarId, now).count()).intValue();
|
||||
}
|
||||
|
||||
/** Returns the first (by event time) poll message not in the future for this registrar. */
|
||||
public static Optional<PollMessage> getFirstPollMessage(String registrarId, DateTime now) {
|
||||
if (tm().isOfy()) {
|
||||
return Optional.ofNullable(datastorePollMessageQuery(registrarId, now).first().now());
|
||||
} else {
|
||||
return jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm()
|
||||
.query(SQL_POLL_MESSAGE_QUERY, PollMessage.class)
|
||||
.setParameter("registrarId", registrarId)
|
||||
.setParameter("now", now)
|
||||
.setMaxResults(1)
|
||||
.getResultStream()
|
||||
.findFirst());
|
||||
}
|
||||
return transactIfJpaTm(
|
||||
() -> createPollMessageQuery(registrarId, now).orderBy("eventTime").first());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -106,14 +78,15 @@ public final class PollFlowUtils {
|
||||
return includeAckedMessageInCount;
|
||||
}
|
||||
|
||||
/** A Datastore query for poll messages from the given registrar that are not in the future. */
|
||||
public static Query<PollMessage> datastorePollMessageQuery(String registrarId, DateTime now) {
|
||||
return ofy()
|
||||
.load()
|
||||
.type(PollMessage.class)
|
||||
.filter("clientId", registrarId)
|
||||
.filter("eventTime <=", now.toDate())
|
||||
.order("eventTime");
|
||||
/**
|
||||
* Returns the QueryComposer for poll messages from the given registrar that are not in the
|
||||
* future.
|
||||
*/
|
||||
public static QueryComposer<PollMessage> createPollMessageQuery(
|
||||
String registrarId, DateTime now) {
|
||||
return tm().createQueryComposer(PollMessage.class)
|
||||
.where("clientId", EQ, registrarId)
|
||||
.where("eventTime", LTE, now);
|
||||
}
|
||||
|
||||
private PollFlowUtils() {}
|
||||
|
||||
@@ -16,24 +16,12 @@ package google.registry.keyring.kms;
|
||||
|
||||
import static com.google.common.base.CaseFormat.LOWER_HYPHEN;
|
||||
import static com.google.common.base.CaseFormat.UPPER_UNDERSCORE;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.keyring.api.KeySerializer;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.keyring.api.KeyringException;
|
||||
import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.server.KmsSecretRevision;
|
||||
import google.registry.model.server.KmsSecretRevisionSqlDao;
|
||||
import google.registry.privileges.secretmanager.KeyringSecretStore;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
import org.bouncycastle.openpgp.PGPException;
|
||||
import org.bouncycastle.openpgp.PGPKeyPair;
|
||||
@@ -47,11 +35,9 @@ import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
* @see <a href="https://cloud.google.com/kms/docs/">Google Cloud Key Management Service
|
||||
* Documentation</a>
|
||||
*/
|
||||
// TODO(2021-06-01): rename this class to SecretManagerKeyring and delete KmsSecretRevision
|
||||
// TODO(2021-07-01): rename this class to SecretManagerKeyring and delete KmsSecretRevision
|
||||
public class KmsKeyring implements Keyring {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
/** Key labels for private key secrets. */
|
||||
enum PrivateKeyLabel {
|
||||
BRDA_SIGNING_PRIVATE,
|
||||
@@ -92,13 +78,10 @@ public class KmsKeyring implements Keyring {
|
||||
}
|
||||
}
|
||||
|
||||
private final KmsConnection kmsConnection;
|
||||
private final KeyringSecretStore secretStore;
|
||||
|
||||
@Inject
|
||||
KmsKeyring(
|
||||
@Config("defaultKmsConnection") KmsConnection kmsConnection, KeyringSecretStore secretStore) {
|
||||
this.kmsConnection = kmsConnection;
|
||||
KmsKeyring(KeyringSecretStore secretStore) {
|
||||
this.secretStore = secretStore;
|
||||
}
|
||||
|
||||
@@ -201,44 +184,11 @@ public class KmsKeyring implements Keyring {
|
||||
return getKeyPair(keyLabel).getPrivateKey();
|
||||
}
|
||||
|
||||
private byte[] getDecryptedDataFromDatastore(String keyName) {
|
||||
String encryptedData;
|
||||
if (tm().isOfy()) {
|
||||
KmsSecret secret =
|
||||
ofy().load().key(Key.create(getCrossTldKey(), KmsSecret.class, keyName)).now();
|
||||
checkState(secret != null, "Requested secret '%s' does not exist.", keyName);
|
||||
encryptedData = ofy().load().key(secret.getLatestRevision()).now().getEncryptedValue();
|
||||
} else {
|
||||
Optional<KmsSecretRevision> revision =
|
||||
tm().transact(() -> KmsSecretRevisionSqlDao.getLatestRevision(keyName));
|
||||
checkState(revision.isPresent(), "Requested secret '%s' does not exist.", keyName);
|
||||
encryptedData = revision.get().getEncryptedValue();
|
||||
}
|
||||
|
||||
try {
|
||||
return kmsConnection.decrypt(keyName, encryptedData);
|
||||
} catch (Exception e) {
|
||||
return new byte[0];
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] getDataFromSecretStore(String keyName) {
|
||||
private byte[] getDecryptedData(String keyName) {
|
||||
try {
|
||||
return secretStore.getSecret(keyName);
|
||||
} catch (Exception e) {
|
||||
throw new KeyringException("Failed to retrieve secret for " + keyName, e);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] getDecryptedData(String keyName) {
|
||||
byte[] dsData = getDecryptedDataFromDatastore(keyName);
|
||||
byte[] secretStoreData = getDataFromSecretStore(keyName);
|
||||
|
||||
if (Arrays.equals(dsData, secretStoreData)) {
|
||||
logger.atInfo().log("Values for %s in Datastore and Secret Manager match.", keyName);
|
||||
return secretStoreData;
|
||||
}
|
||||
logger.atWarning().log("Values for %s in Datastore and Secret Manager do not match.", keyName);
|
||||
return secretStoreData;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,18 +32,13 @@ import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.MARKSDB_SMDR
|
||||
import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.RDE_SSH_CLIENT_PRIVATE_STRING;
|
||||
import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.RDE_SSH_CLIENT_PUBLIC_STRING;
|
||||
import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.SAFE_BROWSING_API_KEY;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.keyring.api.KeySerializer;
|
||||
import google.registry.keyring.kms.KmsKeyring.PrivateKeyLabel;
|
||||
import google.registry.keyring.kms.KmsKeyring.PublicKeyLabel;
|
||||
import google.registry.keyring.kms.KmsKeyring.StringKeyLabel;
|
||||
import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.server.KmsSecretRevision;
|
||||
import google.registry.privileges.secretmanager.KeyringSecretStore;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
@@ -62,14 +57,11 @@ import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
public final class KmsUpdater {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private final KmsConnection kmsConnection;
|
||||
private final KeyringSecretStore secretStore;
|
||||
private final HashMap<String, byte[]> secretValues;
|
||||
|
||||
@Inject
|
||||
public KmsUpdater(
|
||||
@Config("defaultKmsConnection") KmsConnection kmsConnection, KeyringSecretStore secretStore) {
|
||||
this.kmsConnection = kmsConnection;
|
||||
public KmsUpdater(KeyringSecretStore secretStore) {
|
||||
this.secretStore = secretStore;
|
||||
|
||||
// Use LinkedHashMap to preserve insertion order on update() to simplify testing and debugging
|
||||
@@ -150,24 +142,6 @@ public final class KmsUpdater {
|
||||
+ "Please check the status of Secret Manager and re-run the command.",
|
||||
e);
|
||||
}
|
||||
|
||||
// TODO(2021-06-01): remove the writes to Datastore
|
||||
persistEncryptedValues(encryptValues(secretValues));
|
||||
}
|
||||
|
||||
/**
|
||||
* Encrypts updated secrets using KMS. If the configured {@code KeyRing} or {@code CryptoKey}
|
||||
* associated with a secret doesn't exist, they will first be created.
|
||||
*
|
||||
* @see google.registry.config.RegistryConfigSettings.Kms
|
||||
*/
|
||||
private ImmutableMap<String, EncryptResponse> encryptValues(Map<String, byte[]> keyValues) {
|
||||
ImmutableMap.Builder<String, EncryptResponse> encryptedValues = new ImmutableMap.Builder<>();
|
||||
for (Map.Entry<String, byte[]> entry : keyValues.entrySet()) {
|
||||
String secretName = entry.getKey();
|
||||
encryptedValues.put(secretName, kmsConnection.encrypt(secretName, entry.getValue()));
|
||||
}
|
||||
return encryptedValues.build();
|
||||
}
|
||||
|
||||
private KmsUpdater setString(String key, StringKeyLabel stringKeyLabel) {
|
||||
@@ -195,32 +169,6 @@ public final class KmsUpdater {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persists encrypted secrets to Datastore as {@link KmsSecretRevision} entities and makes them
|
||||
* primary. {@link KmsSecret} entities point to the latest {@link KmsSecretRevision}.
|
||||
*
|
||||
* <p>The changes are committed transactionally; if an error occurs, all existing {@link
|
||||
* KmsSecretRevision} entities will remain primary.
|
||||
*/
|
||||
private static void persistEncryptedValues(
|
||||
final ImmutableMap<String, EncryptResponse> encryptedValues) {
|
||||
tm().transact(
|
||||
() -> {
|
||||
for (Map.Entry<String, EncryptResponse> entry : encryptedValues.entrySet()) {
|
||||
String secretName = entry.getKey();
|
||||
EncryptResponse revisionData = entry.getValue();
|
||||
|
||||
KmsSecretRevision secretRevision =
|
||||
new KmsSecretRevision.Builder()
|
||||
.setEncryptedValue(revisionData.ciphertext())
|
||||
.setKmsCryptoKeyVersionName(revisionData.cryptoKeyVersionName())
|
||||
.setParent(secretName)
|
||||
.build();
|
||||
tm().putAll(secretRevision, KmsSecret.create(secretName, secretRevision));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void setSecret(String secretName, byte[] value) {
|
||||
checkArgument(!secretValues.containsKey(secretName), "Attempted to set %s twice", secretName);
|
||||
secretValues.put(secretName, value);
|
||||
|
||||
@@ -16,7 +16,7 @@ package google.registry.model;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
|
||||
import google.registry.model.ofy.ObjectifyService;
|
||||
import google.registry.util.TypeUtils.TypeInstantiator;
|
||||
@@ -57,8 +57,14 @@ public interface Buildable {
|
||||
// If this object has a Long or long Objectify @Id field that is not set, set it now.
|
||||
Field idField = null;
|
||||
try {
|
||||
idField = ModelUtils.getAllFields(instance.getClass()).get(
|
||||
ofy().factory().getMetadata(instance.getClass()).getKeyMetadata().getIdFieldName());
|
||||
idField =
|
||||
ModelUtils.getAllFields(instance.getClass())
|
||||
.get(
|
||||
auditedOfy()
|
||||
.factory()
|
||||
.getMetadata(instance.getClass())
|
||||
.getKeyMetadata()
|
||||
.getIdFieldName());
|
||||
} catch (Exception e) {
|
||||
// Expected if the class is not registered with Objectify.
|
||||
}
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model;
|
||||
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.common.DatabaseTransitionSchedule;
|
||||
import google.registry.model.common.DatabaseTransitionSchedule.PrimaryDatabase;
|
||||
import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
|
||||
|
||||
/** Utility methods related to migrating dual-read/dual-write entities. */
|
||||
public class DatabaseMigrationUtils {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
/** Throws exceptions only in unit tests, otherwise only logs exceptions. */
|
||||
public static void suppressExceptionUnlessInTest(Runnable work, String message) {
|
||||
try {
|
||||
work.run();
|
||||
} catch (Exception e) {
|
||||
if (RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)) {
|
||||
throw e;
|
||||
}
|
||||
logger.atWarning().withCause(e).log(message);
|
||||
}
|
||||
}
|
||||
|
||||
/** Gets the value for the database currently considered primary. */
|
||||
public static PrimaryDatabase getPrimaryDatabase(TransitionId transitionId) {
|
||||
return DatabaseTransitionSchedule.getCached(transitionId)
|
||||
.map(DatabaseTransitionSchedule::getPrimaryDatabase)
|
||||
.orElse(PrimaryDatabase.DATASTORE);
|
||||
}
|
||||
|
||||
public static boolean isDatastore(TransitionId transitionId) {
|
||||
return tm().transactNew(() -> DatabaseMigrationUtils.getPrimaryDatabase(transitionId))
|
||||
.equals(PrimaryDatabase.DATASTORE);
|
||||
}
|
||||
|
||||
private DatabaseMigrationUtils() {}
|
||||
}
|
||||
@@ -17,7 +17,7 @@ package google.registry.model;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.DatabaseTransitionSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.EntityGroupRoot;
|
||||
import google.registry.model.common.GaeUserIdConverter;
|
||||
import google.registry.model.contact.ContactHistory;
|
||||
@@ -47,9 +47,9 @@ import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.server.KmsSecretRevision;
|
||||
import google.registry.model.server.Lock;
|
||||
import google.registry.model.server.ServerSecret;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListRevision;
|
||||
import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListRevision;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
|
||||
import google.registry.model.tmch.TmchCrl;
|
||||
import google.registry.schema.replay.LastSqlTransaction;
|
||||
|
||||
@@ -64,7 +64,7 @@ public final class EntityClasses {
|
||||
BillingEvent.Modification.class,
|
||||
BillingEvent.OneTime.class,
|
||||
BillingEvent.Recurring.class,
|
||||
ClaimsListShard.class,
|
||||
ClaimsList.class,
|
||||
ClaimsListRevision.class,
|
||||
ClaimsListSingleton.class,
|
||||
CommitLogBucket.class,
|
||||
@@ -75,7 +75,7 @@ public final class EntityClasses {
|
||||
ContactHistory.class,
|
||||
ContactResource.class,
|
||||
Cursor.class,
|
||||
DatabaseTransitionSchedule.class,
|
||||
DatabaseMigrationStateSchedule.class,
|
||||
DomainBase.class,
|
||||
DomainHistory.class,
|
||||
EntityGroupRoot.class,
|
||||
|
||||
@@ -155,9 +155,16 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
|
||||
return repoId;
|
||||
}
|
||||
|
||||
/** This method exists solely to satisfy Hibernate. Use {@link Builder} instead. */
|
||||
@SuppressWarnings("UnusedMethod")
|
||||
private void setRepoId(String repoId) {
|
||||
/**
|
||||
* Sets the repository ID.
|
||||
*
|
||||
* <p>This should only be used for restoring the repo id of an object being loaded in a PostLoad
|
||||
* method (effectively, when it is still under construction by Hibernate). In all other cases, the
|
||||
* object should be regarded as immutable and changes should go through a Builder.
|
||||
*
|
||||
* <p>In addition to this special case use, this method must exist to satisfy Hibernate.
|
||||
*/
|
||||
public void setRepoId(String repoId) {
|
||||
this.repoId = repoId;
|
||||
}
|
||||
|
||||
|
||||
@@ -16,10 +16,11 @@ package google.registry.model;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static google.registry.util.DateTimeUtils.latestOf;
|
||||
@@ -28,8 +29,6 @@ import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Result;
|
||||
import com.googlecode.objectify.util.ResultNow;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.EppResource.BuilderWithTransferData;
|
||||
import google.registry.model.EppResource.ForeignKeyedEppResource;
|
||||
@@ -42,14 +41,18 @@ import google.registry.model.index.ForeignKeyIndex;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.reporting.HistoryEntryDao;
|
||||
import google.registry.model.transfer.DomainTransferData;
|
||||
import google.registry.model.transfer.TransferData;
|
||||
import google.registry.model.transfer.TransferStatus;
|
||||
import google.registry.persistence.VKey;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.persistence.Query;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -265,80 +268,147 @@ public final class EppResourceUtils {
|
||||
* Rewinds an {@link EppResource} object to a given point in time.
|
||||
*
|
||||
* <p>This method costs nothing if {@code resource} is already current. Otherwise it needs to
|
||||
* perform a single asynchronous key fetch operation.
|
||||
* perform a single fetch operation.
|
||||
*
|
||||
* <p><b>Warning:</b> A resource can only be rolled backwards in time, not forwards; therefore
|
||||
* {@code resource} should be whatever's currently in Datastore.
|
||||
*
|
||||
* <p><b>Warning:</b> Revisions are granular to 24-hour periods. It's recommended that
|
||||
* {@code timestamp} be set to midnight. Otherwise you must take into consideration that under
|
||||
* certain circumstances, a resource might be restored to a revision on the previous day, even if
|
||||
* there were revisions made earlier on the same date as {@code timestamp}; however, a resource
|
||||
* will never be restored to a revision occurring after {@code timestamp}. This behavior is due to
|
||||
* the way {@link google.registry.model.translators.CommitLogRevisionsTranslatorFactory
|
||||
* <p><b>Warning:</b> In Datastore, revisions are granular to 24-hour periods. It's recommended
|
||||
* that {@code timestamp} be set to midnight. If you don't use midnight, you must take into
|
||||
* consideration that under certain circumstances, a resource might be restored to a revision on
|
||||
* the previous day, even if there were revisions made earlier on the same date as {@code
|
||||
* timestamp}; however, a resource will never be restored to a revision occurring after {@code
|
||||
* timestamp}. This behavior is due to the way {@link
|
||||
* google.registry.model.translators.CommitLogRevisionsTranslatorFactory
|
||||
* CommitLogRevisionsTranslatorFactory} manages the {@link EppResource#revisions} field. Please
|
||||
* note however that the creation and deletion times of a resource are granular to the
|
||||
* millisecond.
|
||||
*
|
||||
* @return an asynchronous operation returning resource at {@code timestamp} or {@code null} if
|
||||
* resource is deleted or not yet created
|
||||
* <p>Example: a resource in Datastore has three revisions A, B, and C
|
||||
*
|
||||
* <ul>
|
||||
* <li>A: Day 0, 1pm
|
||||
* <li>B: Day 1, 1pm
|
||||
* <li>C: Day 1, 3pm
|
||||
* </ul>
|
||||
*
|
||||
* <p>If one requests the resource as of day 1 at 2pm, we will return revision A because as far as
|
||||
* the commit logs are concerned, revision C completely overwrites the existence of revision B.
|
||||
*
|
||||
* <p>When using the SQL backend (post-Registry-3.0-migration) this restriction goes away and
|
||||
* objects can be restored to any revision.
|
||||
*
|
||||
* @return the resource at {@code timestamp} or {@code null} if resource is deleted or not yet
|
||||
* created
|
||||
*/
|
||||
public static <T extends EppResource>
|
||||
Result<T> loadAtPointInTime(final T resource, final DateTime timestamp) {
|
||||
public static <T extends EppResource> T loadAtPointInTime(
|
||||
final T resource, final DateTime timestamp) {
|
||||
// If we're before the resource creation time, don't try to find a "most recent revision".
|
||||
if (timestamp.isBefore(resource.getCreationTime())) {
|
||||
return new ResultNow<>(null);
|
||||
return null;
|
||||
}
|
||||
// If the resource was not modified after the requested time, then use it as-is, otherwise find
|
||||
// the most recent revision asynchronously, and return an async result that wraps that revision
|
||||
// and returns it projected forward to exactly the desired timestamp, or null if the resource is
|
||||
// deleted at that timestamp.
|
||||
final Result<T> loadResult =
|
||||
// the most recent revision and project it forward to exactly the desired timestamp, or null if
|
||||
// the resource is deleted at that timestamp.
|
||||
T loadedResource =
|
||||
isAtOrAfter(timestamp, resource.getUpdateTimestamp().getTimestamp())
|
||||
? new ResultNow<>(resource)
|
||||
? resource
|
||||
: loadMostRecentRevisionAtTime(resource, timestamp);
|
||||
return () -> {
|
||||
T loadedResource = loadResult.now();
|
||||
return (loadedResource == null) ? null
|
||||
: (isActive(loadedResource, timestamp)
|
||||
? cloneProjectedAtTime(loadedResource, timestamp)
|
||||
: null);
|
||||
};
|
||||
return (loadedResource == null)
|
||||
? null
|
||||
: (isActive(loadedResource, timestamp)
|
||||
? cloneProjectedAtTime(loadedResource, timestamp)
|
||||
: null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an asynchronous result holding the most recent Datastore revision of a given
|
||||
* EppResource before or at the provided timestamp using the EppResource revisions map, falling
|
||||
* back to using the earliest revision or the resource as-is if there are no revisions.
|
||||
* Rewinds an {@link EppResource} object to a given point in time.
|
||||
*
|
||||
* <p>This method costs nothing if {@code resource} is already current. Otherwise it returns an
|
||||
* async operation that performs a single fetch operation.
|
||||
*
|
||||
* @return an asynchronous operation returning resource at {@code timestamp} or {@code null} if
|
||||
* resource is deleted or not yet created
|
||||
* @see #loadAtPointInTime(EppResource, DateTime)
|
||||
*/
|
||||
public static <T extends EppResource> Supplier<T> loadAtPointInTimeAsync(
|
||||
final T resource, final DateTime timestamp) {
|
||||
return () -> loadAtPointInTime(resource, timestamp);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent revision of a given EppResource before or at the provided timestamp,
|
||||
* falling back to using the resource as-is if there are no revisions.
|
||||
*
|
||||
* @see #loadAtPointInTime(EppResource, DateTime)
|
||||
*/
|
||||
private static <T extends EppResource> Result<T> loadMostRecentRevisionAtTime(
|
||||
private static <T extends EppResource> T loadMostRecentRevisionAtTime(
|
||||
final T resource, final DateTime timestamp) {
|
||||
if (tm().isOfy()) {
|
||||
return loadMostRecentRevisionAtTimeDatastore(resource, timestamp);
|
||||
} else {
|
||||
return loadMostRecentRevisionAtTimeSql(resource, timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent Datastore revision of a given EppResource before or at the provided
|
||||
* timestamp using the EppResource revisions map, falling back to using the resource as-is if
|
||||
* there are no revisions.
|
||||
*
|
||||
* @see #loadAtPointInTimeAsync(EppResource, DateTime)
|
||||
*/
|
||||
private static <T extends EppResource> T loadMostRecentRevisionAtTimeDatastore(
|
||||
final T resource, final DateTime timestamp) {
|
||||
final Key<T> resourceKey = Key.create(resource);
|
||||
final Key<CommitLogManifest> revision = findMostRecentRevisionAtTime(resource, timestamp);
|
||||
final Key<CommitLogManifest> revision =
|
||||
findMostRecentDatastoreRevisionAtTime(resource, timestamp);
|
||||
if (revision == null) {
|
||||
logger.atSevere().log("No revision found for %s, falling back to resource.", resourceKey);
|
||||
return new ResultNow<>(resource);
|
||||
}
|
||||
final Result<CommitLogMutation> mutationResult =
|
||||
ofy().load().key(CommitLogMutation.createKey(revision, resourceKey));
|
||||
return () -> {
|
||||
CommitLogMutation mutation = mutationResult.now();
|
||||
if (mutation != null) {
|
||||
return ofy().load().fromEntity(mutation.getEntity());
|
||||
}
|
||||
logger.atSevere().log(
|
||||
"Couldn't load mutation for revision at %s for %s, falling back to resource."
|
||||
+ " Revision: %s",
|
||||
timestamp, resourceKey, revision);
|
||||
return resource;
|
||||
};
|
||||
}
|
||||
final CommitLogMutation mutation =
|
||||
auditedOfy().load().key(CommitLogMutation.createKey(revision, resourceKey)).now();
|
||||
if (mutation != null) {
|
||||
return auditedOfy().load().fromEntity(mutation.getEntity());
|
||||
}
|
||||
logger.atSevere().log(
|
||||
"Couldn't load mutation for revision at %s for %s, falling back to resource."
|
||||
+ " Revision: %s",
|
||||
timestamp, resourceKey, revision);
|
||||
return resource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most recent SQL revision of a given EppResource before or at the provided timestamp
|
||||
* using *History objects, falling back to using the resource as-is if there are no revisions.
|
||||
*
|
||||
* @see #loadAtPointInTimeAsync(EppResource, DateTime)
|
||||
*/
|
||||
private static <T extends EppResource> T loadMostRecentRevisionAtTimeSql(
|
||||
T resource, DateTime timestamp) {
|
||||
@SuppressWarnings("unchecked")
|
||||
T resourceAtPointInTime =
|
||||
(T)
|
||||
HistoryEntryDao.loadHistoryObjectsForResource(
|
||||
resource.createVKey(), START_OF_TIME, timestamp)
|
||||
.stream()
|
||||
.max(Comparator.comparing(HistoryEntry::getModificationTime))
|
||||
.flatMap(HistoryEntry::getResourceAtPointInTime)
|
||||
.orElse(null);
|
||||
if (resourceAtPointInTime == null) {
|
||||
logger.atSevere().log(
|
||||
"Couldn't load resource at % for key %s, falling back to resource %s.",
|
||||
timestamp, resource.createVKey(), resource);
|
||||
return resource;
|
||||
}
|
||||
return resourceAtPointInTime;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private static <T extends EppResource> Key<CommitLogManifest>
|
||||
findMostRecentRevisionAtTime(final T resource, final DateTime timestamp) {
|
||||
private static <T extends EppResource>
|
||||
Key<CommitLogManifest> findMostRecentDatastoreRevisionAtTime(
|
||||
final T resource, final DateTime timestamp) {
|
||||
final Key<T> resourceKey = Key.create(resource);
|
||||
Entry<?, Key<CommitLogManifest>> revision = resource.getRevisions().floorEntry(timestamp);
|
||||
if (revision != null) {
|
||||
@@ -366,27 +436,26 @@ public final class EppResourceUtils {
|
||||
*
|
||||
* @param key the referent key
|
||||
* @param now the logical time of the check
|
||||
* @param limit the maximum number of returned keys
|
||||
* @param limit the maximum number of returned keys, unlimited if null
|
||||
*/
|
||||
public static ImmutableSet<VKey<DomainBase>> getLinkedDomainKeys(
|
||||
VKey<? extends EppResource> key, DateTime now, int limit) {
|
||||
VKey<? extends EppResource> key, DateTime now, @Nullable Integer limit) {
|
||||
checkArgument(
|
||||
key.getKind().equals(ContactResource.class) || key.getKind().equals(HostResource.class),
|
||||
"key must be either VKey<ContactResource> or VKey<HostResource>, but it is %s",
|
||||
key);
|
||||
boolean isContactKey = key.getKind().equals(ContactResource.class);
|
||||
if (tm().isOfy()) {
|
||||
return ofy()
|
||||
.load()
|
||||
.type(DomainBase.class)
|
||||
.filter(isContactKey ? "allContacts.contact" : "nsHosts", key.getOfyKey())
|
||||
.filter("deletionTime >", now)
|
||||
.limit(limit)
|
||||
.keys()
|
||||
.list()
|
||||
.stream()
|
||||
.map(DomainBase::createVKey)
|
||||
.collect(toImmutableSet());
|
||||
com.googlecode.objectify.cmd.Query<DomainBase> query =
|
||||
auditedOfy()
|
||||
.load()
|
||||
.type(DomainBase.class)
|
||||
.filter(isContactKey ? "allContacts.contact" : "nsHosts", key.getOfyKey())
|
||||
.filter("deletionTime >", now);
|
||||
if (limit != null) {
|
||||
query.limit(limit);
|
||||
}
|
||||
return query.keys().list().stream().map(DomainBase::createVKey).collect(toImmutableSet());
|
||||
} else {
|
||||
return tm().transact(
|
||||
() -> {
|
||||
@@ -405,15 +474,20 @@ public final class EppResourceUtils {
|
||||
.setParameter("fkRepoId", key.getSqlKey())
|
||||
.setParameter("now", now.toDate());
|
||||
}
|
||||
return (ImmutableSet<VKey<DomainBase>>)
|
||||
query
|
||||
.setMaxResults(limit)
|
||||
.getResultStream()
|
||||
.map(
|
||||
repoId ->
|
||||
DomainBase.createVKey(
|
||||
Key.create(DomainBase.class, (String) repoId)))
|
||||
.collect(toImmutableSet());
|
||||
if (limit != null) {
|
||||
query.setMaxResults(limit);
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
ImmutableSet<VKey<DomainBase>> domainBaseKeySet =
|
||||
(ImmutableSet<VKey<DomainBase>>)
|
||||
query
|
||||
.getResultStream()
|
||||
.map(
|
||||
repoId ->
|
||||
DomainBase.createVKey(
|
||||
Key.create(DomainBase.class, (String) repoId)))
|
||||
.collect(toImmutableSet());
|
||||
return domainBaseKeySet;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ package google.registry.model;
|
||||
|
||||
import static com.google.common.collect.Iterables.transform;
|
||||
import static com.google.common.collect.Maps.transformValues;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static java.lang.annotation.ElementType.FIELD;
|
||||
import static java.lang.annotation.RetentionPolicy.RUNTIME;
|
||||
@@ -189,7 +189,7 @@ public abstract class ImmutableObject implements Cloneable {
|
||||
private static Object hydrate(Object value) {
|
||||
if (value instanceof Key) {
|
||||
if (tm().isOfy()) {
|
||||
return hydrate(ofy().load().key((Key<?>) value).now());
|
||||
return hydrate(auditedOfy().load().key((Key<?>) value).now());
|
||||
}
|
||||
return value;
|
||||
} else if (value instanceof Map) {
|
||||
|
||||
@@ -39,8 +39,8 @@ import google.registry.model.registrar.RegistrarContact;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.Registry.TldState;
|
||||
import google.registry.model.registry.label.PremiumList;
|
||||
import google.registry.model.registry.label.PremiumListDualDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.schema.tld.PremiumListDao;
|
||||
import google.registry.util.CidrAddressBlock;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
@@ -295,7 +295,7 @@ public final class OteAccountBuilder {
|
||||
boolean isEarlyAccess,
|
||||
int roidSuffix) {
|
||||
String tldNameAlphaNumerical = tldName.replaceAll("[^a-z0-9]", "");
|
||||
Optional<PremiumList> premiumList = PremiumListDualDao.getLatestRevision(DEFAULT_PREMIUM_LIST);
|
||||
Optional<PremiumList> premiumList = PremiumListDao.getLatestRevision(DEFAULT_PREMIUM_LIST);
|
||||
checkState(premiumList.isPresent(), "Couldn't find premium list %s.", DEFAULT_PREMIUM_LIST);
|
||||
Registry.Builder builder =
|
||||
new Registry.Builder()
|
||||
|
||||
@@ -46,7 +46,6 @@ import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.GracePeriod;
|
||||
import google.registry.model.domain.rgp.GracePeriodStatus;
|
||||
import google.registry.model.domain.token.AllocationToken;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
|
||||
import google.registry.persistence.BillingVKey.BillingEventVKey;
|
||||
import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
|
||||
@@ -115,7 +114,7 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
/** Entity id. */
|
||||
@Id @javax.persistence.Id Long id;
|
||||
|
||||
@Parent @DoNotHydrate @Transient Key<? extends HistoryEntry> parent;
|
||||
@Parent @DoNotHydrate @Transient Key<DomainHistory> parent;
|
||||
|
||||
/** The registrar to bill. */
|
||||
@Index
|
||||
@@ -154,7 +153,7 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
parent =
|
||||
Key.create(
|
||||
Key.create(DomainBase.class, domainRepoId),
|
||||
HistoryEntry.class,
|
||||
DomainHistory.class,
|
||||
domainHistoryRevisionId);
|
||||
}
|
||||
|
||||
@@ -192,7 +191,7 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
return targetId;
|
||||
}
|
||||
|
||||
public Key<? extends HistoryEntry> getParentKey() {
|
||||
public Key<DomainHistory> getParentKey() {
|
||||
return parent;
|
||||
}
|
||||
|
||||
@@ -254,12 +253,12 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
return thisCastToDerived();
|
||||
}
|
||||
|
||||
public B setParent(HistoryEntry parent) {
|
||||
public B setParent(DomainHistory parent) {
|
||||
getInstance().parent = Key.create(parent);
|
||||
return thisCastToDerived();
|
||||
}
|
||||
|
||||
public B setParent(Key<? extends HistoryEntry> parentKey) {
|
||||
public B setParent(Key<DomainHistory> parentKey) {
|
||||
getInstance().parent = parentKey;
|
||||
return thisCastToDerived();
|
||||
}
|
||||
@@ -325,13 +324,22 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
DateTime syntheticCreationTime;
|
||||
|
||||
/**
|
||||
* For {@link Flag#SYNTHETIC} events, a {@link Key} to the {@link BillingEvent} from which this
|
||||
* OneTime was created. This is needed in order to properly match billing events against {@link
|
||||
* Cancellation}s.
|
||||
* For {@link Flag#SYNTHETIC} events, a {@link Key} to the {@link Recurring} from which this
|
||||
* {@link OneTime} was created. This is needed in order to properly match billing events against
|
||||
* {@link Cancellation}s.
|
||||
*/
|
||||
@Column(name = "cancellation_matching_billing_recurrence_id")
|
||||
VKey<Recurring> cancellationMatchingBillingEvent;
|
||||
|
||||
/**
|
||||
* For {@link Flag#SYNTHETIC} events, the {@link DomainHistory} revision ID of the the {@link
|
||||
* Recurring} from which this {@link OneTime} was created. This is needed in order to recreate
|
||||
* the {@link VKey} when reading from SQL.
|
||||
*/
|
||||
@Ignore
|
||||
@Column(name = "recurrence_history_revision_id")
|
||||
Long recurringEventHistoryRevisionId;
|
||||
|
||||
/**
|
||||
* The {@link AllocationToken} used in the creation of this event, or null if one was not used.
|
||||
*/
|
||||
@@ -355,10 +363,14 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
return syntheticCreationTime;
|
||||
}
|
||||
|
||||
public VKey<? extends BillingEvent> getCancellationMatchingBillingEvent() {
|
||||
public VKey<Recurring> getCancellationMatchingBillingEvent() {
|
||||
return cancellationMatchingBillingEvent;
|
||||
}
|
||||
|
||||
public Long getRecurringEventHistoryRevisionId() {
|
||||
return recurringEventHistoryRevisionId;
|
||||
}
|
||||
|
||||
public Optional<VKey<AllocationToken>> getAllocationToken() {
|
||||
return Optional.ofNullable(allocationToken);
|
||||
}
|
||||
@@ -377,6 +389,28 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
return new Builder(clone(this));
|
||||
}
|
||||
|
||||
@Override
|
||||
void onLoad() {
|
||||
super.onLoad();
|
||||
if (cancellationMatchingBillingEvent != null) {
|
||||
recurringEventHistoryRevisionId =
|
||||
cancellationMatchingBillingEvent.getOfyKey().getParent().getId();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void postLoad() {
|
||||
super.postLoad();
|
||||
if (cancellationMatchingBillingEvent != null) {
|
||||
cancellationMatchingBillingEvent =
|
||||
cancellationMatchingBillingEvent.restoreOfy(
|
||||
DomainBase.class,
|
||||
domainRepoId,
|
||||
DomainHistory.class,
|
||||
recurringEventHistoryRevisionId);
|
||||
}
|
||||
}
|
||||
|
||||
/** A builder for {@link OneTime} since it is immutable. */
|
||||
public static class Builder extends BillingEvent.Builder<OneTime, Builder> {
|
||||
|
||||
@@ -411,6 +445,8 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
public Builder setCancellationMatchingBillingEvent(
|
||||
VKey<Recurring> cancellationMatchingBillingEvent) {
|
||||
getInstance().cancellationMatchingBillingEvent = cancellationMatchingBillingEvent;
|
||||
getInstance().recurringEventHistoryRevisionId =
|
||||
cancellationMatchingBillingEvent.getOfyKey().getParent().getId();
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -445,6 +481,11 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
== (instance.cancellationMatchingBillingEvent != null),
|
||||
"Cancellation matching billing event must be set if and only if the SYNTHETIC flag "
|
||||
+ "is set.");
|
||||
checkState(
|
||||
!instance.getFlags().contains(Flag.SYNTHETIC)
|
||||
|| (instance.cancellationMatchingBillingEvent.getOfyKey().getParent().getId()
|
||||
== instance.recurringEventHistoryRevisionId),
|
||||
"Cancellation matching billing event and its history revision ID does not match.");
|
||||
return super.build();
|
||||
}
|
||||
}
|
||||
@@ -735,7 +776,7 @@ public abstract class BillingEvent extends ImmutableObject
|
||||
* because it is needed by one-off scrap tools that need to make billing adjustments.
|
||||
*/
|
||||
public static Modification createRefundFor(
|
||||
OneTime billingEvent, HistoryEntry historyEntry, String description) {
|
||||
OneTime billingEvent, DomainHistory historyEntry, String description) {
|
||||
return new Builder()
|
||||
.setClientId(billingEvent.getClientId())
|
||||
.setFlags(billingEvent.getFlags())
|
||||
|
||||
@@ -17,7 +17,6 @@ package google.registry.model.common;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.base.Splitter;
|
||||
@@ -210,12 +209,7 @@ public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
|
||||
private static void checkValidCursorTypeForScope(
|
||||
CursorType cursorType, Key<? extends ImmutableObject> scope) {
|
||||
checkArgument(
|
||||
cursorType
|
||||
.getScopeClass()
|
||||
.equals(
|
||||
scope.equals(getCrossTldKey())
|
||||
? EntityGroupRoot.class
|
||||
: ofy().factory().getMetadata(scope).getEntityClass()),
|
||||
cursorType.getScopeClass().getSimpleName().equals(scope.getKind()),
|
||||
"Class required for cursor does not match scope class");
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.common;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import com.googlecode.objectify.annotation.Embed;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Mapify;
|
||||
import google.registry.model.annotations.InCrossTld;
|
||||
import google.registry.model.common.TimedTransitionProperty.TimeMapper;
|
||||
import google.registry.model.common.TimedTransitionProperty.TimedTransition;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
import java.time.Duration;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* A wrapper object representing the stage-to-time mapping of the Registry 3.0 Cloud SQL migration.
|
||||
*
|
||||
* <p>The entity is stored in Datastore throughout the entire migration so as to have a single point
|
||||
* of access.
|
||||
*/
|
||||
@Entity
|
||||
@InCrossTld
|
||||
public class DatabaseMigrationStateSchedule extends CrossTldSingleton
|
||||
implements DatastoreOnlyEntity {
|
||||
|
||||
public enum PrimaryDatabase {
|
||||
CLOUD_SQL,
|
||||
DATASTORE
|
||||
}
|
||||
|
||||
public enum ReplayDirection {
|
||||
NO_REPLAY,
|
||||
DATASTORE_TO_SQL,
|
||||
SQL_TO_DATASTORE
|
||||
}
|
||||
|
||||
/**
|
||||
* The current phase of the migration plus information about which database to use and whether or
|
||||
* not the phase is read-only.
|
||||
*/
|
||||
public enum MigrationState {
|
||||
DATASTORE_ONLY(PrimaryDatabase.DATASTORE, false, ReplayDirection.NO_REPLAY),
|
||||
DATASTORE_PRIMARY(PrimaryDatabase.DATASTORE, false, ReplayDirection.DATASTORE_TO_SQL),
|
||||
DATASTORE_PRIMARY_READ_ONLY(PrimaryDatabase.DATASTORE, true, ReplayDirection.DATASTORE_TO_SQL),
|
||||
SQL_PRIMARY_READ_ONLY(PrimaryDatabase.CLOUD_SQL, true, ReplayDirection.SQL_TO_DATASTORE),
|
||||
SQL_PRIMARY(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.SQL_TO_DATASTORE),
|
||||
SQL_ONLY(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.NO_REPLAY);
|
||||
|
||||
private final PrimaryDatabase primaryDatabase;
|
||||
private final boolean isReadOnly;
|
||||
private final ReplayDirection replayDirection;
|
||||
|
||||
public PrimaryDatabase getPrimaryDatabase() {
|
||||
return primaryDatabase;
|
||||
}
|
||||
|
||||
public boolean isReadOnly() {
|
||||
return isReadOnly;
|
||||
}
|
||||
|
||||
public ReplayDirection getReplayDirection() {
|
||||
return replayDirection;
|
||||
}
|
||||
|
||||
MigrationState(
|
||||
PrimaryDatabase primaryDatabase, boolean isReadOnly, ReplayDirection replayDirection) {
|
||||
this.primaryDatabase = primaryDatabase;
|
||||
this.isReadOnly = isReadOnly;
|
||||
this.replayDirection = replayDirection;
|
||||
}
|
||||
}
|
||||
|
||||
@Embed
|
||||
public static class MigrationStateTransition extends TimedTransition<MigrationState> {
|
||||
private MigrationState migrationState;
|
||||
|
||||
@Override
|
||||
protected MigrationState getValue() {
|
||||
return migrationState;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setValue(MigrationState migrationState) {
|
||||
this.migrationState = migrationState;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache of the current migration schedule. The key is meaningless; this is essentially a memoized
|
||||
* Supplier that can be reset for testing purposes and after writes.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
public static final LoadingCache<
|
||||
Class<DatabaseMigrationStateSchedule>,
|
||||
TimedTransitionProperty<MigrationState, MigrationStateTransition>>
|
||||
// Each instance should cache the migration schedule for five minutes before reloading
|
||||
CACHE =
|
||||
CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(Duration.ofMinutes(5))
|
||||
.build(
|
||||
new CacheLoader<
|
||||
Class<DatabaseMigrationStateSchedule>,
|
||||
TimedTransitionProperty<MigrationState, MigrationStateTransition>>() {
|
||||
@Override
|
||||
public TimedTransitionProperty<MigrationState, MigrationStateTransition> load(
|
||||
Class<DatabaseMigrationStateSchedule> unused) {
|
||||
return DatabaseMigrationStateSchedule.getUncached();
|
||||
}
|
||||
});
|
||||
|
||||
// Restrictions on the state transitions, e.g. no going from DATASTORE_ONLY to SQL_ONLY
|
||||
private static final ImmutableMultimap<MigrationState, MigrationState> VALID_STATE_TRANSITIONS =
|
||||
createValidStateTransitions();
|
||||
|
||||
/**
|
||||
* The valid state transitions. Generally, one can advance the state one step or move backward any
|
||||
* number of steps, as long as the step we're moving back to has the same primary database as the
|
||||
* one we're in. Otherwise, we must move to the corresponding READ_ONLY stage first.
|
||||
*/
|
||||
private static ImmutableMultimap<MigrationState, MigrationState> createValidStateTransitions() {
|
||||
ImmutableMultimap.Builder<MigrationState, MigrationState> builder =
|
||||
new ImmutableMultimap.Builder<MigrationState, MigrationState>()
|
||||
.put(MigrationState.DATASTORE_ONLY, MigrationState.DATASTORE_PRIMARY)
|
||||
.putAll(
|
||||
MigrationState.DATASTORE_PRIMARY,
|
||||
MigrationState.DATASTORE_ONLY,
|
||||
MigrationState.DATASTORE_PRIMARY_READ_ONLY)
|
||||
.putAll(
|
||||
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
|
||||
MigrationState.DATASTORE_ONLY,
|
||||
MigrationState.DATASTORE_PRIMARY,
|
||||
MigrationState.SQL_PRIMARY_READ_ONLY,
|
||||
MigrationState.SQL_PRIMARY)
|
||||
.putAll(
|
||||
MigrationState.SQL_PRIMARY_READ_ONLY,
|
||||
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
|
||||
MigrationState.SQL_PRIMARY)
|
||||
.putAll(
|
||||
MigrationState.SQL_PRIMARY,
|
||||
MigrationState.SQL_PRIMARY_READ_ONLY,
|
||||
MigrationState.SQL_ONLY)
|
||||
.putAll(
|
||||
MigrationState.SQL_ONLY,
|
||||
MigrationState.SQL_PRIMARY_READ_ONLY,
|
||||
MigrationState.SQL_PRIMARY);
|
||||
// In addition, we can always transition from a state to itself (useful when updating the map).
|
||||
for (MigrationState migrationState : MigrationState.values()) {
|
||||
builder.put(migrationState, migrationState);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// Default map to return if we have never saved any -- only use Datastore.
|
||||
@VisibleForTesting
|
||||
public static final TimedTransitionProperty<MigrationState, MigrationStateTransition>
|
||||
DEFAULT_TRANSITION_MAP =
|
||||
TimedTransitionProperty.fromValueMap(
|
||||
ImmutableSortedMap.of(START_OF_TIME, MigrationState.DATASTORE_ONLY),
|
||||
MigrationStateTransition.class);
|
||||
|
||||
@VisibleForTesting
|
||||
@Mapify(TimeMapper.class)
|
||||
public TimedTransitionProperty<MigrationState, MigrationStateTransition> migrationTransitions =
|
||||
TimedTransitionProperty.forMapify(
|
||||
MigrationState.DATASTORE_ONLY, MigrationStateTransition.class);
|
||||
|
||||
// Required for Objectify initialization
|
||||
private DatabaseMigrationStateSchedule() {}
|
||||
|
||||
@VisibleForTesting
|
||||
DatabaseMigrationStateSchedule(
|
||||
TimedTransitionProperty<MigrationState, MigrationStateTransition> migrationTransitions) {
|
||||
this.migrationTransitions = migrationTransitions;
|
||||
}
|
||||
|
||||
/** Sets and persists to Datastore the provided migration transition schedule. */
|
||||
public static void set(ImmutableSortedMap<DateTime, MigrationState> migrationTransitionMap) {
|
||||
ofyTm().assertInTransaction();
|
||||
TimedTransitionProperty<MigrationState, MigrationStateTransition> transitions =
|
||||
TimedTransitionProperty.make(
|
||||
migrationTransitionMap,
|
||||
MigrationStateTransition.class,
|
||||
VALID_STATE_TRANSITIONS,
|
||||
"validStateTransitions",
|
||||
MigrationState.DATASTORE_ONLY,
|
||||
"migrationTransitionMap must start with DATASTORE_ONLY");
|
||||
validateTransitionAtCurrentTime(transitions);
|
||||
ofyTm().put(new DatabaseMigrationStateSchedule(transitions));
|
||||
CACHE.invalidateAll();
|
||||
}
|
||||
|
||||
/** Loads the currently-set migration schedule from the cache, or the default if none exists. */
|
||||
public static TimedTransitionProperty<MigrationState, MigrationStateTransition> get() {
|
||||
return CACHE.getUnchecked(DatabaseMigrationStateSchedule.class);
|
||||
}
|
||||
|
||||
/** Returns the database migration status at the given time. */
|
||||
public static MigrationState getValueAtTime(DateTime dateTime) {
|
||||
return get().getValueAtTime(dateTime);
|
||||
}
|
||||
|
||||
/** Loads the currently-set migration schedule from Datastore, or the default if none exists. */
|
||||
@VisibleForTesting
|
||||
static TimedTransitionProperty<MigrationState, MigrationStateTransition> getUncached() {
|
||||
return ofyTm()
|
||||
.transact(
|
||||
() ->
|
||||
ofyTm()
|
||||
.loadSingleton(DatabaseMigrationStateSchedule.class)
|
||||
.map(s -> s.migrationTransitions)
|
||||
.orElse(DEFAULT_TRANSITION_MAP));
|
||||
}
|
||||
|
||||
/**
|
||||
* A provided map of transitions may be valid by itself (i.e. it shifts states properly, doesn't
|
||||
* skip states, and doesn't backtrack incorrectly) while still being invalid. In addition to the
|
||||
* transitions in the map being valid, the single transition from the current map at the current
|
||||
* time to the new map at the current time time must also be valid.
|
||||
*/
|
||||
private static void validateTransitionAtCurrentTime(
|
||||
TimedTransitionProperty<MigrationState, MigrationStateTransition> newTransitions) {
|
||||
MigrationState currentValue = getUncached().getValueAtTime(ofyTm().getTransactionTime());
|
||||
MigrationState nextCurrentValue = newTransitions.getValueAtTime(ofyTm().getTransactionTime());
|
||||
checkArgument(
|
||||
VALID_STATE_TRANSITIONS.get(currentValue).contains(nextCurrentValue),
|
||||
"Cannot transition from current state-as-of-now %s to new state-as-of-now %s",
|
||||
currentValue,
|
||||
nextCurrentValue);
|
||||
}
|
||||
}
|
||||
@@ -1,166 +0,0 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.common;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static google.registry.config.RegistryConfig.getSingletonCacheRefreshDuration;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Embed;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Mapify;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.UpdateAutoTimestamp;
|
||||
import google.registry.model.annotations.InCrossTld;
|
||||
import google.registry.model.common.TimedTransitionProperty.TimeMapper;
|
||||
import google.registry.model.common.TimedTransitionProperty.TimedTransition;
|
||||
import google.registry.model.registry.label.PremiumList;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.smd.SignedMarkRevocationList;
|
||||
import google.registry.model.tmch.ClaimsListShard;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
import java.util.Optional;
|
||||
import javax.annotation.concurrent.Immutable;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
@Entity
|
||||
@Immutable
|
||||
@InCrossTld
|
||||
public class DatabaseTransitionSchedule extends ImmutableObject implements DatastoreOnlyEntity {
|
||||
|
||||
/**
|
||||
* The name of the database to be treated as the primary database. The first entry in the schedule
|
||||
* will always be Datastore.
|
||||
*/
|
||||
public enum PrimaryDatabase {
|
||||
CLOUD_SQL,
|
||||
DATASTORE
|
||||
}
|
||||
|
||||
/** The id of the transition schedule. */
|
||||
public enum TransitionId {
|
||||
/** The schedule for migration of {@link ClaimsListShard} entities. */
|
||||
CLAIMS_LIST,
|
||||
/** The schedule for the migration of {@link PremiumList} and {@link ReservedList}. */
|
||||
DOMAIN_LABEL_LISTS,
|
||||
/** The schedule for the migration of the {@link SignedMarkRevocationList} entity. */
|
||||
SIGNED_MARK_REVOCATION_LIST,
|
||||
/** The schedule for all asynchronously-replayed entities, ones not dually-written. */
|
||||
REPLAYED_ENTITIES,
|
||||
}
|
||||
|
||||
/**
|
||||
* The transition to a specified primary database at a specific point in time, for use in a
|
||||
* TimedTransitionProperty.
|
||||
*/
|
||||
@Embed
|
||||
public static class PrimaryDatabaseTransition extends TimedTransition<PrimaryDatabase> {
|
||||
private PrimaryDatabase primaryDatabase;
|
||||
|
||||
@Override
|
||||
protected PrimaryDatabase getValue() {
|
||||
return primaryDatabase;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setValue(PrimaryDatabase primaryDatabase) {
|
||||
this.primaryDatabase = primaryDatabase;
|
||||
}
|
||||
}
|
||||
|
||||
@Parent Key<EntityGroupRoot> parent = getCrossTldKey();
|
||||
|
||||
@Id String transitionId;
|
||||
|
||||
/** An automatically managed timestamp of when this schedule was last written to Datastore. */
|
||||
UpdateAutoTimestamp lastUpdateTime = UpdateAutoTimestamp.create(null);
|
||||
|
||||
/** A property that tracks the primary database for a dual-read/dual-write database migration. */
|
||||
@Mapify(TimeMapper.class)
|
||||
TimedTransitionProperty<PrimaryDatabase, PrimaryDatabaseTransition> databaseTransitions =
|
||||
TimedTransitionProperty.forMapify(PrimaryDatabase.DATASTORE, PrimaryDatabaseTransition.class);
|
||||
|
||||
/** A cache that loads the {@link DatabaseTransitionSchedule} for a given id. */
|
||||
private static final LoadingCache<TransitionId, Optional<DatabaseTransitionSchedule>> CACHE =
|
||||
CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(
|
||||
java.time.Duration.ofMillis(getSingletonCacheRefreshDuration().getMillis()))
|
||||
.build(
|
||||
new CacheLoader<TransitionId, Optional<DatabaseTransitionSchedule>>() {
|
||||
@Override
|
||||
public Optional<DatabaseTransitionSchedule> load(TransitionId transitionId) {
|
||||
return DatabaseTransitionSchedule.get(transitionId);
|
||||
}
|
||||
});
|
||||
|
||||
public static DatabaseTransitionSchedule create(
|
||||
TransitionId transitionId,
|
||||
TimedTransitionProperty<PrimaryDatabase, PrimaryDatabaseTransition> databaseTransitions) {
|
||||
checkNotNull(transitionId, "Id cannot be null");
|
||||
checkNotNull(databaseTransitions, "databaseTransitions cannot be null");
|
||||
databaseTransitions.checkValidity();
|
||||
DatabaseTransitionSchedule instance = new DatabaseTransitionSchedule();
|
||||
instance.transitionId = transitionId.name();
|
||||
instance.databaseTransitions = databaseTransitions;
|
||||
return instance;
|
||||
}
|
||||
|
||||
/** Returns the database that is indicated as primary at the given time. */
|
||||
public PrimaryDatabase getPrimaryDatabase() {
|
||||
return databaseTransitions.getValueAtTime(tm().getTransactionTime());
|
||||
}
|
||||
|
||||
/** Returns the database transitions as a map of start time to primary database. */
|
||||
public ImmutableSortedMap<DateTime, PrimaryDatabase> getDatabaseTransitions() {
|
||||
return databaseTransitions.toValueMap();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current cached schedule for the given id.
|
||||
*
|
||||
* <p>WARNING: The schedule returned by this method could be up to 10 minutes out of date.
|
||||
*/
|
||||
public static Optional<DatabaseTransitionSchedule> getCached(TransitionId id) {
|
||||
return CACHE.getUnchecked(id);
|
||||
}
|
||||
|
||||
/** Returns the schedule for a given id. */
|
||||
public static Optional<DatabaseTransitionSchedule> get(TransitionId transitionId) {
|
||||
VKey<DatabaseTransitionSchedule> key =
|
||||
VKey.create(
|
||||
DatabaseTransitionSchedule.class,
|
||||
transitionId,
|
||||
Key.create(getCrossTldKey(), DatabaseTransitionSchedule.class, transitionId.name()));
|
||||
|
||||
return ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format(
|
||||
"%s(last updated at %s): %s",
|
||||
transitionId, lastUpdateTime.getTimestamp(), databaseTransitions.toValueMap());
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.EntitySubclass;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.contact.ContactHistory.ContactHistoryId;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
@@ -41,6 +42,11 @@ import javax.persistence.PostLoad;
|
||||
* <p>In addition to the general history fields (e.g. action time, registrar ID) we also persist a
|
||||
* copy of the contact entity at this point in time. We persist a raw {@link ContactBase} so that
|
||||
* the foreign-keyed fields in that class can refer to this object.
|
||||
*
|
||||
* <p>This class is only marked as a Datastore entity subclass and registered with Objectify so that
|
||||
* when building it its ID can be auto-populated by Objectify. It is converted to its superclass
|
||||
* {@link HistoryEntry} when persisted to Datastore using {@link
|
||||
* google.registry.persistence.transaction.TransactionManager}.
|
||||
*/
|
||||
@Entity
|
||||
@javax.persistence.Table(
|
||||
@@ -102,6 +108,11 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
|
||||
return (VKey<ContactHistory>) createVKey(Key.create(this));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<? extends EppResource> getResourceAtPointInTime() {
|
||||
return getContactBase();
|
||||
}
|
||||
|
||||
@PostLoad
|
||||
void postLoad() {
|
||||
// Normally Hibernate would see that the contact fields are all null and would fill contactBase
|
||||
@@ -110,7 +121,10 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
|
||||
contactBase = null;
|
||||
}
|
||||
if (contactBase != null && contactBase.getRepoId() == null) {
|
||||
contactBase = contactBase.asBuilder().setRepoId(parent.getName()).build();
|
||||
// contactBase hasn't been fully constructed yet, so it's ok to go in and mutate it. Though
|
||||
// the use of the Builder is not necessarily problematic in this case, this is still safer as
|
||||
// the Builder can do things like comparisons that compute the hash code.
|
||||
contactBase.setRepoId(parent.getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,9 @@ import javax.persistence.Index;
|
||||
import javax.persistence.JoinColumn;
|
||||
import javax.persistence.JoinTable;
|
||||
import javax.persistence.OneToMany;
|
||||
import javax.persistence.PostLoad;
|
||||
import javax.persistence.Table;
|
||||
import org.hibernate.Hibernate;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
@@ -54,12 +56,16 @@ import org.joda.time.DateTime;
|
||||
@Table(
|
||||
name = "Domain",
|
||||
indexes = {
|
||||
@Index(columnList = "adminContact"),
|
||||
@Index(columnList = "autorenewEndTime"),
|
||||
@Index(columnList = "billingContact"),
|
||||
@Index(columnList = "creationTime"),
|
||||
@Index(columnList = "currentSponsorRegistrarId"),
|
||||
@Index(columnList = "deletionTime"),
|
||||
@Index(columnList = "domainName"),
|
||||
@Index(columnList = "techContact"),
|
||||
@Index(columnList = "tld"),
|
||||
@Index(columnList = "autorenewEndTime")
|
||||
@Index(columnList = "registrantContact")
|
||||
})
|
||||
@WithStringVKey
|
||||
@ExternalMessagingName("domain")
|
||||
@@ -74,6 +80,8 @@ public class DomainBase extends DomainContent
|
||||
return super.getRepoId();
|
||||
}
|
||||
|
||||
// It seems like this should be FetchType.EAGER, but for some reason when we do that we get a lazy
|
||||
// load error during the load of a domain.
|
||||
@ElementCollection
|
||||
@JoinTable(
|
||||
name = "DomainHost",
|
||||
@@ -135,6 +143,16 @@ public class DomainBase extends DomainContent
|
||||
return dsData;
|
||||
}
|
||||
|
||||
/** Post-load method to eager load the collections. */
|
||||
@PostLoad
|
||||
@Override
|
||||
protected void postLoad() {
|
||||
super.postLoad();
|
||||
// TODO(b/188044616): Determine why Eager loading doesn't work here.
|
||||
Hibernate.initialize(dsData);
|
||||
Hibernate.initialize(gracePeriods);
|
||||
}
|
||||
|
||||
@Override
|
||||
public VKey<DomainBase> createVKey() {
|
||||
return VKey.create(DomainBase.class, getRepoId(), Key.create(this));
|
||||
|
||||
@@ -173,8 +173,9 @@ public class DomainContent extends EppResource
|
||||
@Transient Set<DelegationSignerData> dsData;
|
||||
|
||||
/**
|
||||
* The claims notice supplied when this application or domain was created, if there was one. It's
|
||||
* {@literal @}XmlTransient because it's not returned in an info response.
|
||||
* The claims notice supplied when this domain was created, if there was one.
|
||||
*
|
||||
* <p>It's {@literal @}XmlTransient because it's not returned in an info response.
|
||||
*/
|
||||
@IgnoreSave(IfNull.class)
|
||||
@Embedded
|
||||
@@ -337,8 +338,7 @@ public class DomainContent extends EppResource
|
||||
}
|
||||
|
||||
@PostLoad
|
||||
@SuppressWarnings("UnusedMethod")
|
||||
private void postLoad() {
|
||||
protected void postLoad() {
|
||||
// Reconstitute the contact list.
|
||||
ImmutableSet.Builder<DesignatedContact> contactsBuilder = new ImmutableSet.Builder<>();
|
||||
|
||||
@@ -706,7 +706,7 @@ public class DomainContent extends EppResource
|
||||
return authInfo;
|
||||
}
|
||||
|
||||
/** Returns all referenced contacts from this domain or application. */
|
||||
/** Returns all referenced contacts from this domain. */
|
||||
public ImmutableSet<VKey<ContactResource>> getReferencedContacts() {
|
||||
return nullToEmptyImmutableCopy(allContacts).stream()
|
||||
.map(DesignatedContact::getContactKey)
|
||||
|
||||
@@ -21,10 +21,11 @@ import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.EntitySubclass;
|
||||
import com.googlecode.objectify.annotation.Ignore;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.domain.GracePeriod.GracePeriodHistory;
|
||||
import google.registry.model.domain.secdns.DelegationSignerData;
|
||||
import google.registry.model.domain.secdns.DomainDsDataHistory;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.reporting.DomainTransactionRecord;
|
||||
@@ -55,6 +56,7 @@ import javax.persistence.JoinTable;
|
||||
import javax.persistence.OneToMany;
|
||||
import javax.persistence.PostLoad;
|
||||
import javax.persistence.Table;
|
||||
import org.hibernate.Hibernate;
|
||||
|
||||
/**
|
||||
* A persisted history entry representing an EPP modification to a domain.
|
||||
@@ -62,6 +64,11 @@ import javax.persistence.Table;
|
||||
* <p>In addition to the general history fields (e.g. action time, registrar ID) we also persist a
|
||||
* copy of the domain entity at this point in time. We persist a raw {@link DomainContent} so that
|
||||
* the foreign-keyed fields in that class can refer to this object.
|
||||
*
|
||||
* <p>This class is only marked as a Datastore entity subclass and registered with Objectify so that
|
||||
* when building it its ID can be auto-populated by Objectify. It is converted to its superclass
|
||||
* {@link HistoryEntry} when persisted to Datastore using {@link
|
||||
* google.registry.persistence.transaction.TransactionManager}.
|
||||
*/
|
||||
@Entity
|
||||
@Table(
|
||||
@@ -97,7 +104,6 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
|
||||
// We could have reused domainContent.nsHosts here, but Hibernate throws a weird exception after
|
||||
// we change to use a composite primary key.
|
||||
// TODO(b/166776754): Investigate if we can reuse domainContent.nsHosts for storing host keys.
|
||||
@Ignore
|
||||
@ElementCollection
|
||||
@JoinTable(
|
||||
name = "DomainHistoryHost",
|
||||
@@ -111,7 +117,6 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
|
||||
@Column(name = "host_repo_id")
|
||||
Set<VKey<HostResource>> nsHosts;
|
||||
|
||||
@Ignore
|
||||
@OneToMany(
|
||||
cascade = {CascadeType.ALL},
|
||||
fetch = FetchType.EAGER,
|
||||
@@ -131,7 +136,6 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
|
||||
// HashSet rather than ImmutableSet so that Hibernate can fill them out lazily on request
|
||||
Set<DomainDsDataHistory> dsDataHistories = new HashSet<>();
|
||||
|
||||
@Ignore
|
||||
@OneToMany(
|
||||
cascade = {CascadeType.ALL},
|
||||
fetch = FetchType.EAGER,
|
||||
@@ -246,20 +250,40 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
|
||||
return (VKey<DomainHistory>) createVKey(Key.create(this));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<? extends EppResource> getResourceAtPointInTime() {
|
||||
return getDomainContent();
|
||||
}
|
||||
|
||||
@PostLoad
|
||||
void postLoad() {
|
||||
if (domainContent != null) {
|
||||
domainContent.nsHosts = nullToEmptyImmutableCopy(nsHosts);
|
||||
domainContent.gracePeriods =
|
||||
gracePeriodHistories.stream()
|
||||
.map(GracePeriod::createFromHistory)
|
||||
.collect(toImmutableSet());
|
||||
domainContent.dsData =
|
||||
dsDataHistories.stream().map(DelegationSignerData::create).collect(toImmutableSet());
|
||||
// Normally Hibernate would see that the domain fields are all null and would fill
|
||||
// domainContent with a null object. Unfortunately, the updateTimestamp is never null in SQL.
|
||||
if (domainContent.getDomainName() == null) {
|
||||
domainContent = null;
|
||||
} else {
|
||||
if (domainContent.getRepoId() == null) {
|
||||
domainContent = domainContent.asBuilder().setRepoId(parent.getName()).build();
|
||||
// domainContent still hasn't been fully constructed yet, so it's ok to go in and mutate
|
||||
// it. In fact, we have to because going through the builder causes the hash codes of
|
||||
// contained objects to be calculated prematurely.
|
||||
domainContent.setRepoId(parent.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(b/188044616): Determine why Eager loading doesn't work here.
|
||||
Hibernate.initialize(domainTransactionRecords);
|
||||
Hibernate.initialize(nsHosts);
|
||||
Hibernate.initialize(dsDataHistories);
|
||||
Hibernate.initialize(gracePeriodHistories);
|
||||
}
|
||||
|
||||
// In Datastore, save as a HistoryEntry object regardless of this object's type
|
||||
|
||||
@@ -115,6 +115,17 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
|
||||
type, domainRepoId, expirationTime, clientId, billingEventOneTime, null, gracePeriodId);
|
||||
}
|
||||
|
||||
public static GracePeriod createFromHistory(GracePeriodHistory history) {
|
||||
return createInternal(
|
||||
history.type,
|
||||
history.domainRepoId,
|
||||
history.expirationTime,
|
||||
history.clientId,
|
||||
history.billingEventOneTime == null ? null : history.billingEventOneTime.createVKey(),
|
||||
history.billingEventRecurring == null ? null : history.billingEventRecurring.createVKey(),
|
||||
history.gracePeriodId);
|
||||
}
|
||||
|
||||
/** Creates a GracePeriod for a Recurring billing event. */
|
||||
public static GracePeriod createForRecurring(
|
||||
GracePeriodStatus type,
|
||||
|
||||
@@ -51,8 +51,8 @@ import javax.xml.bind.annotation.XmlValue;
|
||||
public class LaunchPhase extends ImmutableObject {
|
||||
|
||||
/**
|
||||
* The phase during which trademark holders can submit registrations or applications with
|
||||
* trademark information that can be validated by the server.
|
||||
* The phase during which trademark holders can submit domain registrations with trademark
|
||||
* information that can be validated by the server.
|
||||
*/
|
||||
public static final LaunchPhase SUNRISE = create("sunrise");
|
||||
|
||||
|
||||
@@ -114,6 +114,15 @@ public class DelegationSignerData extends DomainDsDataBase {
|
||||
return create(keyTag, algorithm, digestType, DatatypeConverter.parseHexBinary(digestAsHex));
|
||||
}
|
||||
|
||||
public static DelegationSignerData create(DomainDsDataHistory history) {
|
||||
return create(
|
||||
history.keyTag,
|
||||
history.algorithm,
|
||||
history.digestType,
|
||||
history.digest,
|
||||
history.domainRepoId);
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link DelegationSignerData} entity. */
|
||||
static class DomainDsDataId extends ImmutableObject implements Serializable {
|
||||
|
||||
|
||||
@@ -16,9 +16,7 @@ package google.registry.model.domain.secdns;
|
||||
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.ofy.ObjectifyService;
|
||||
import google.registry.schema.replay.DatastoreEntity;
|
||||
import google.registry.schema.replay.SqlEntity;
|
||||
import java.util.Optional;
|
||||
import google.registry.schema.replay.SqlOnlyEntity;
|
||||
import javax.persistence.Access;
|
||||
import javax.persistence.AccessType;
|
||||
import javax.persistence.Column;
|
||||
@@ -27,7 +25,7 @@ import javax.persistence.Id;
|
||||
|
||||
/** Entity class to represent a historic {@link DelegationSignerData}. */
|
||||
@Entity
|
||||
public class DomainDsDataHistory extends DomainDsDataBase implements SqlEntity {
|
||||
public class DomainDsDataHistory extends DomainDsDataBase implements SqlOnlyEntity {
|
||||
|
||||
@Id Long dsDataHistoryRevisionId;
|
||||
|
||||
@@ -84,9 +82,4 @@ public class DomainDsDataHistory extends DomainDsDataBase implements SqlEntity {
|
||||
public byte[] getDigest() {
|
||||
return super.getDigest();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<DatastoreEntity> toDatastoreEntity() {
|
||||
return Optional.empty(); // Not persisted in Datastore
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.EntitySubclass;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.host.HostHistory.HostHistoryId;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
@@ -41,6 +42,11 @@ import javax.persistence.PostLoad;
|
||||
* <p>In addition to the general history fields (e.g. action time, registrar ID) we also persist a
|
||||
* copy of the host entity at this point in time. We persist a raw {@link HostBase} so that the
|
||||
* foreign-keyed fields in that class can refer to this object.
|
||||
*
|
||||
* <p>This class is only marked as a Datastore entity subclass and registered with Objectify so that
|
||||
* when building it its ID can be auto-populated by Objectify. It is converted to its superclass
|
||||
* {@link HistoryEntry} when persisted to Datastore using {@link
|
||||
* google.registry.persistence.transaction.TransactionManager}.
|
||||
*/
|
||||
@Entity
|
||||
@javax.persistence.Table(
|
||||
@@ -103,6 +109,11 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
|
||||
return (VKey<HostHistory>) createVKey(Key.create(this));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<? extends EppResource> getResourceAtPointInTime() {
|
||||
return getHostBase();
|
||||
}
|
||||
|
||||
@PostLoad
|
||||
void postLoad() {
|
||||
// Normally Hibernate would see that the host fields are all null and would fill hostBase
|
||||
@@ -111,7 +122,10 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
|
||||
hostBase = null;
|
||||
}
|
||||
if (hostBase != null && hostBase.getRepoId() == null) {
|
||||
hostBase = hostBase.asBuilder().setRepoId(parent.getName()).build();
|
||||
// hostBase hasn't been fully constructed yet, so it's ok to go in and mutate it. Though the
|
||||
// use of the Builder is not necessarily problematic in this case, this is still safer as the
|
||||
// Builder can do things like comparisons that compute the hash code.
|
||||
hostBase.setRepoId(parent.getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
|
||||
import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.util.CollectionUtils.entriesToImmutableMap;
|
||||
@@ -220,16 +220,15 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
|
||||
Class<ForeignKeyIndex<E>> fkiClass = mapToFkiClass(clazz);
|
||||
return ImmutableMap.copyOf(
|
||||
inTransaction
|
||||
? ofy().load().type(fkiClass).ids(foreignKeys)
|
||||
: tm().doTransactionless(() -> ofy().load().type(fkiClass).ids(foreignKeys)));
|
||||
? auditedOfy().load().type(fkiClass).ids(foreignKeys)
|
||||
: tm().doTransactionless(() -> auditedOfy().load().type(fkiClass).ids(foreignKeys)));
|
||||
} else {
|
||||
String property = RESOURCE_CLASS_TO_FKI_PROPERTY.get(clazz);
|
||||
ImmutableList<ForeignKeyIndex<E>> indexes =
|
||||
tm().transact(
|
||||
() ->
|
||||
jpaTm()
|
||||
.getEntityManager()
|
||||
.createQuery(
|
||||
.query(
|
||||
CriteriaQueryBuilder.create(clazz)
|
||||
.whereFieldIsIn(property, foreignKeys)
|
||||
.build())
|
||||
@@ -276,7 +275,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
|
||||
ImmutableSet<VKey<ForeignKeyIndex<?>>> typedKeys = ImmutableSet.copyOf(keys);
|
||||
ImmutableMap<String, ? extends ForeignKeyIndex<? extends EppResource>> existingFkis =
|
||||
loadIndexesFromStore(resourceClass, foreignKeys, false);
|
||||
// ofy() omits keys that don't have values in Datastore, so re-add them in
|
||||
// ofy omits keys that don't have values in Datastore, so re-add them in
|
||||
// here with Optional.empty() values.
|
||||
return Maps.asMap(
|
||||
typedKeys,
|
||||
|
||||
@@ -27,7 +27,7 @@ import java.util.Arrays;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* A Deleter that forwards to {@code ofy().delete()}, but can be augmented via subclassing to
|
||||
* A Deleter that forwards to {@code auditedOfy().delete()}, but can be augmented via subclassing to
|
||||
* do custom processing on the keys to be deleted prior to their deletion.
|
||||
*/
|
||||
abstract class AugmentedDeleter implements Deleter {
|
||||
|
||||
@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.DiscreteDomain.integers;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.config.RegistryConfig.getCommitLogBucketCount;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.collect.ContiguousSet;
|
||||
@@ -118,7 +118,7 @@ public class CommitLogBucket extends ImmutableObject implements Buildable, Datas
|
||||
|
||||
/** Returns the loaded bucket for the given key, or a new object if the bucket doesn't exist. */
|
||||
public static CommitLogBucket loadBucket(Key<CommitLogBucket> bucketKey) {
|
||||
CommitLogBucket bucket = ofy().load().key(bucketKey).now();
|
||||
CommitLogBucket bucket = auditedOfy().load().key(bucketKey).now();
|
||||
return (bucket == null)
|
||||
? new CommitLogBucket.Builder().setBucketNum(bucketKey.getId()).build()
|
||||
: bucket;
|
||||
@@ -126,7 +126,7 @@ public class CommitLogBucket extends ImmutableObject implements Buildable, Datas
|
||||
|
||||
/** Returns the set of all loaded commit log buckets, filling in missing buckets with new ones. */
|
||||
public static ImmutableSet<CommitLogBucket> loadAllBuckets() {
|
||||
ofy().load().keys(getAllBucketKeys()); // Load all buckets into session cache at once.
|
||||
auditedOfy().load().keys(getAllBucketKeys()); // Load all buckets into session cache at once.
|
||||
ImmutableSet.Builder<CommitLogBucket> allBuckets = new ImmutableSet.Builder<>();
|
||||
for (Key<CommitLogBucket> key : getAllBucketKeys()) {
|
||||
allBuckets.add(loadBucket(key));
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
package google.registry.model.ofy;
|
||||
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
@@ -49,7 +49,7 @@ public class CommitLogCheckpointRoot extends ImmutableObject implements Datastor
|
||||
}
|
||||
|
||||
public static CommitLogCheckpointRoot loadRoot() {
|
||||
CommitLogCheckpointRoot root = ofy().load().key(getKey()).now();
|
||||
CommitLogCheckpointRoot root = auditedOfy().load().key(getKey()).now();
|
||||
return root == null ? new CommitLogCheckpointRoot() : root;
|
||||
}
|
||||
|
||||
|
||||
@@ -14,10 +14,11 @@
|
||||
|
||||
package google.registry.model.ofy;
|
||||
|
||||
|
||||
import static com.google.appengine.api.datastore.EntityTranslator.convertToPb;
|
||||
import static com.google.appengine.api.datastore.EntityTranslator.createFromPbBytes;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
|
||||
import com.google.appengine.api.datastore.KeyFactory;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
@@ -66,7 +67,7 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
|
||||
* converted to a raw Datastore Entity, serialized to bytes, and stored within the mutation.
|
||||
*/
|
||||
public static CommitLogMutation create(Key<CommitLogManifest> parent, Object entity) {
|
||||
return createFromRaw(parent, ofy().save().toEntity(entity));
|
||||
return createFromRaw(parent, auditedOfy().save().toEntity(entity));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -22,7 +22,7 @@ import static com.google.common.collect.Maps.filterKeys;
|
||||
import static com.google.common.collect.Sets.difference;
|
||||
import static com.google.common.collect.Sets.union;
|
||||
import static google.registry.model.ofy.CommitLogBucket.loadBucket;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
@@ -134,7 +134,7 @@ class CommitLoggedWork<R> implements Runnable {
|
||||
// asynchronous save and delete operations that haven't been reaped, but that's ok because we
|
||||
// already logged all of those keys in {@link TransactionInfo} and now just need to figure out
|
||||
// what was loaded.
|
||||
ImmutableSet<Key<?>> keysInSessionCache = ofy().getSessionKeys();
|
||||
ImmutableSet<Key<?>> keysInSessionCache = auditedOfy().getSessionKeys();
|
||||
Map<Key<BackupGroupRoot>, BackupGroupRoot> rootsForTouchedKeys =
|
||||
getBackupGroupRoots(touchedKeys);
|
||||
Map<Key<BackupGroupRoot>, BackupGroupRoot> rootsForUntouchedKeys =
|
||||
@@ -153,14 +153,16 @@ class CommitLoggedWork<R> implements Runnable {
|
||||
.stream()
|
||||
.map(entity -> (ImmutableObject) CommitLogMutation.create(manifestKey, entity))
|
||||
.collect(toImmutableSet());
|
||||
ofy().saveWithoutBackup()
|
||||
.entities(new ImmutableSet.Builder<>()
|
||||
.add(manifest)
|
||||
.add(bucket.asBuilder().setLastWrittenTime(info.transactionTime).build())
|
||||
.addAll(mutations)
|
||||
.addAll(untouchedRootsWithTouchedChildren)
|
||||
.build())
|
||||
.now();
|
||||
auditedOfy()
|
||||
.saveWithoutBackup()
|
||||
.entities(
|
||||
new ImmutableSet.Builder<>()
|
||||
.add(manifest)
|
||||
.add(bucket.asBuilder().setLastWrittenTime(info.transactionTime).build())
|
||||
.addAll(mutations)
|
||||
.addAll(untouchedRootsWithTouchedChildren)
|
||||
.build())
|
||||
.now();
|
||||
ReplayQueue.addInTests(info);
|
||||
}
|
||||
|
||||
@@ -185,8 +187,8 @@ class CommitLoggedWork<R> implements Runnable {
|
||||
Set<Key<BackupGroupRoot>> rootKeys = new HashSet<>();
|
||||
for (Key<?> key : keys) {
|
||||
while (key != null
|
||||
&& !BackupGroupRoot.class
|
||||
.isAssignableFrom(ofy().factory().getMetadata(key).getEntityClass())) {
|
||||
&& !BackupGroupRoot.class.isAssignableFrom(
|
||||
auditedOfy().factory().getMetadata(key).getEntityClass())) {
|
||||
key = key.getParent();
|
||||
}
|
||||
if (key != null) {
|
||||
@@ -195,6 +197,6 @@ class CommitLoggedWork<R> implements Runnable {
|
||||
rootKeys.add(rootKey);
|
||||
}
|
||||
}
|
||||
return ImmutableMap.copyOf(ofy().load().keys(rootKeys));
|
||||
return ImmutableMap.copyOf(auditedOfy().load().keys(rootKeys));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.ImmutableMap.toImmutableMap;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
|
||||
import com.google.common.base.Functions;
|
||||
@@ -64,7 +64,7 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
}
|
||||
|
||||
private Ofy getOfy() {
|
||||
return injectedOfy == null ? ofy() : injectedOfy;
|
||||
return injectedOfy == null ? auditedOfy() : injectedOfy;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -173,6 +173,11 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
putAll(entities);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateAll(Object... entities) {
|
||||
updateAll(ImmutableList.of(entities));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateWithoutBackup(Object entity) {
|
||||
putWithoutBackup(entity);
|
||||
@@ -217,9 +222,17 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
entry -> keyMap.get(entry.getKey()), entry -> toSqlEntity(entry.getValue())));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public <T> ImmutableList<T> loadByEntitiesIfPresent(Iterable<T> entities) {
|
||||
return ImmutableList.copyOf(getOfy().load().entities(entities).values());
|
||||
return getOfy()
|
||||
.load()
|
||||
.entities(toDatastoreEntities(ImmutableList.copyOf(entities)))
|
||||
.values()
|
||||
.stream()
|
||||
.map(DatastoreTransactionManager::toSqlEntity)
|
||||
.map(entity -> (T) entity)
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -245,9 +258,10 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
return result;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public <T> T loadByEntity(T entity) {
|
||||
return (T) toSqlEntity(ofy().load().entity(toDatastoreEntity(entity)).now());
|
||||
return (T) toSqlEntity(auditedOfy().load().entity(toDatastoreEntity(entity)).now());
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -262,13 +276,17 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
|
||||
@Override
|
||||
public <T> ImmutableList<T> loadAllOf(Class<T> clazz) {
|
||||
Query<T> query = getOfy().load().type(clazz);
|
||||
// If the entity is in the cross-TLD entity group, then we can take advantage of an ancestor
|
||||
// query to give us strong transactional consistency.
|
||||
if (clazz.isAnnotationPresent(InCrossTld.class)) {
|
||||
query = query.ancestor(getCrossTldKey());
|
||||
}
|
||||
return ImmutableList.copyOf(query);
|
||||
return ImmutableList.copyOf(getPossibleAncestorQuery(clazz));
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> Optional<T> loadSingleton(Class<T> clazz) {
|
||||
List<T> elements = getPossibleAncestorQuery(clazz).limit(2).list();
|
||||
checkArgument(
|
||||
elements.size() <= 1,
|
||||
"Expected at most one entity of type %s, found at least two",
|
||||
clazz.getSimpleName());
|
||||
return elements.stream().findFirst();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -288,8 +306,9 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(Object entity) {
|
||||
public <T> T delete(T entity) {
|
||||
syncIfTransactionless(getOfy().delete().entity(toDatastoreEntity(entity)));
|
||||
return entity;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -401,6 +420,17 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
return obj;
|
||||
}
|
||||
|
||||
/** A query for returning any/all results of an object, with an ancestor if possible. */
|
||||
private <T> Query<T> getPossibleAncestorQuery(Class<T> clazz) {
|
||||
Query<T> query = getOfy().load().type(clazz);
|
||||
// If the entity is in the cross-TLD entity group, then we can take advantage of an ancestor
|
||||
// query to give us strong transactional consistency.
|
||||
if (clazz.isAnnotationPresent(InCrossTld.class)) {
|
||||
query = query.ancestor(getCrossTldKey());
|
||||
}
|
||||
return query;
|
||||
}
|
||||
|
||||
private static class DatastoreQueryComposerImpl<T> extends QueryComposer<T> {
|
||||
|
||||
DatastoreQueryComposerImpl(Class<T> entityClass) {
|
||||
@@ -409,9 +439,14 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
|
||||
Query<T> buildQuery() {
|
||||
checkOnlyOneInequalityField();
|
||||
Query<T> result = ofy().load().type(entityClass);
|
||||
Query<T> result = auditedOfy().load().type(entityClass);
|
||||
for (WhereClause pred : predicates) {
|
||||
result = result.filter(pred.fieldName + pred.comparator.getDatastoreString(), pred.value);
|
||||
String comparatorString = pred.comparator.getDatastoreString();
|
||||
if (comparatorString == null) {
|
||||
throw new UnsupportedOperationException(
|
||||
String.format("The %s operation is not supported on Datastore.", pred.comparator));
|
||||
}
|
||||
result = result.filter(pred.fieldName + comparatorString, pred.value);
|
||||
}
|
||||
|
||||
if (orderBy != null) {
|
||||
@@ -445,12 +480,18 @@ public class DatastoreTransactionManager implements TransactionManager {
|
||||
|
||||
@Override
|
||||
public long count() {
|
||||
return buildQuery().count();
|
||||
// Objectify provides a count() function, but unfortunately that doesn't work if there are
|
||||
// more than 1000 (the default response page size?) entries in the result set. We also use
|
||||
// chunkAll() here as it provides a nice performance boost.
|
||||
//
|
||||
// There is some information on this issue on SO, see:
|
||||
// https://stackoverflow.com/questions/751124/how-does-one-get-a-count-of-rows-in-a-datastore-model-in-google-app-engine
|
||||
return Iterables.size(buildQuery().chunkAll().keys());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<T> list() {
|
||||
return buildQuery().list();
|
||||
public ImmutableList<T> list() {
|
||||
return ImmutableList.copyOf(buildQuery().list());
|
||||
}
|
||||
|
||||
private void checkOnlyOneInequalityField() {
|
||||
|
||||
@@ -51,7 +51,7 @@ import google.registry.model.translators.VKeyTranslatorFactory;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* An instance of Ofy, obtained via {@code #ofy()}, should be used to access all persistable
|
||||
* An instance of Ofy, obtained via {@code #auditedOfy()}, should be used to access all persistable
|
||||
* objects. The class contains a static initializer to call factory().register(...) on all
|
||||
* persistable objects in this package.
|
||||
*/
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
package google.registry.model.ofy;
|
||||
|
||||
import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
@@ -53,7 +53,7 @@ public class ReplayQueue {
|
||||
// not been applied. Converting the object to an entity and then back again performs
|
||||
// those transformations so that we persist the same values to SQL that we have in
|
||||
// Datastore.
|
||||
builder.put(entry.getKey(), ofy().toPojo(ofy().toEntity(entry.getValue())));
|
||||
builder.put(entry.getKey(), auditedOfy().toPojo(auditedOfy().toEntity(entry.getValue())));
|
||||
}
|
||||
}
|
||||
queue.add(builder.build());
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user