mirror of
https://github.com/google/nomulus
synced 2026-07-07 16:46:56 +00:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Lai Jiang
|
761ae612fd | ||
 gbrodman
|
e2fa60a9c6 | ||
 sarahcaseybot
|
b04dfbf740 | ||
 Weimin Yu
|
a1668ceafd | ||
|
Lai Jiang
|
406d49ac99 | ||
|
Weimin Yu
|
f7b82bc190 | ||
|
Lai Jiang
|
45c398149b | ||
|
gbrodman
|
183c0653fb | ||
|
gbrodman
|
fb002953c8 | ||
 Rachel Guan
|
a369e57e5c | ||
|
gbrodman
|
f32d80fb9d | ||
|
gbrodman
|
afa5a353f1 | ||
|
Rachel Guan
|
c4c5ac85da | ||
 Ben McIlwain
|
4d0078607f | ||
|
Rachel Guan
|
2b78433682 | ||
|
Weimin Yu
|
a0fcd02ed2 | ||
|
Rachel Guan
|
58e413af89 | ||
|
gbrodman
|
38c8e81690 | ||
|
Rachel Guan
|
3beb207fcc | ||
|
gbrodman
|
8cf88b7e18 | ||
|
gbrodman
|
6ec2e9501d |
@@ -0,0 +1,4 @@
|
||||
To report a security issue, please use http://g.co/vulnz. We use
|
||||
http://g.co/vulnz for our intake, and do coordination and disclosure here on
|
||||
GitHub (including using GitHub Security Advisory). The Google Security Team will
|
||||
respond within 5 working days of your report on g.co/vulnz.
|
||||
@@ -100,6 +100,15 @@ public class DateTimeUtils {
|
||||
return ZonedDateTime.ofInstant(instant, ZoneId.of(dateTime.getZone().getID()).normalized());
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a Joda {@link DateTime} object to an equivalent java.time {@link ZonedDateTime}
|
||||
* object.
|
||||
*/
|
||||
public static ZonedDateTime toZonedDateTime(DateTime dateTime, ZoneId zoneId) {
|
||||
java.time.Instant instant = java.time.Instant.ofEpochMilli(dateTime.getMillis());
|
||||
return ZonedDateTime.ofInstant(instant, zoneId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a java.time {@link ZonedDateTime} object to an equivalent Joda {@link DateTime}
|
||||
* object.
|
||||
|
||||
@@ -74,8 +74,6 @@ def fragileTestPatterns = [
|
||||
"google/registry/cron/TldFanoutActionTest.*",
|
||||
// Test Datastore inexplicably aborts transaction.
|
||||
"google/registry/model/tmch/ClaimsListShardTest.*",
|
||||
// Creates large object (64MBytes), occasionally throws OOM error.
|
||||
"google/registry/model/server/KmsSecretRevisionTest.*",
|
||||
// Changes cache timeouts and for some reason appears to have contention
|
||||
// with other tests.
|
||||
"google/registry/whois/WhoisCommandFactoryTest.*",
|
||||
|
||||
@@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -56,10 +56,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -98,16 +98,16 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
@@ -138,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -163,7 +163,7 @@ io.github.classgraph:classgraph:4.8.102
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
|
||||
@@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -97,16 +97,16 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
@@ -136,8 +136,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -161,7 +161,7 @@ io.github.classgraph:classgraph:4.8.102
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
|
||||
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -60,10 +60,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -103,16 +103,16 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
@@ -143,8 +143,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -172,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
|
||||
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
||||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api-client:google-api-client:1.32.1
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
@@ -60,10 +60,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:api-common:1.10.4
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.api:gax-httpjson:0.83.0
|
||||
com.google.api:gax:1.66.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
@@ -103,16 +103,16 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
||||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-core-http:1.95.4
|
||||
com.google.cloud:google-cloud-core:1.95.4
|
||||
com.google.cloud:google-cloud-nio:0.123.4
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.cloud:google-cloud-storage:1.118.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.code.gson:gson:2.8.7
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
@@ -143,8 +143,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.protobuf:protobuf-java-util:3.17.3
|
||||
com.google.protobuf:protobuf-java:3.17.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
@@ -172,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
|
||||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-context:1.39.0
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
@@ -23,13 +24,13 @@ import static google.registry.util.DateTimeUtils.latestOf;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.common.util.concurrent.ListeningExecutorService;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import dagger.Lazy;
|
||||
import google.registry.backup.BackupModule.Backups;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import java.io.IOException;
|
||||
@@ -39,6 +40,7 @@ import java.util.TreeMap;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Provider;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Utility class to list commit logs diff files stored on GCS. */
|
||||
@@ -51,7 +53,7 @@ class GcsDiffFileLister {
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Backups Lazy<ListeningExecutorService> lazyExecutor;
|
||||
@Inject @Backups Provider<ListeningExecutorService> executorProvider;
|
||||
@Inject ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
@Inject
|
||||
@@ -113,22 +115,27 @@ class GcsDiffFileLister {
|
||||
// (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
|
||||
// an upper checkpoint time > fromTime.
|
||||
TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
|
||||
ImmutableList<String> strippedFilenames;
|
||||
String commitLogDiffPrefix = getCommitLogDiffPrefix(fromTime, toTime);
|
||||
ImmutableList<String> filenames;
|
||||
try {
|
||||
strippedFilenames = gcsUtils.listFolderObjects(gcsBucket, DIFF_FILE_PREFIX);
|
||||
filenames =
|
||||
gcsUtils.listFolderObjects(gcsBucket, commitLogDiffPrefix).stream()
|
||||
.map(s -> commitLogDiffPrefix + s)
|
||||
.collect(toImmutableList());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
DateTime lastUpperBoundTime = START_OF_TIME;
|
||||
|
||||
TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
|
||||
ListeningExecutorService executor = executorProvider.get();
|
||||
try {
|
||||
for (String strippedFilename : strippedFilenames) {
|
||||
final String filename = DIFF_FILE_PREFIX + strippedFilename;
|
||||
for (String filename : filenames) {
|
||||
String strippedFilename = filename.replaceFirst(DIFF_FILE_PREFIX, "");
|
||||
DateTime upperBoundTime = DateTime.parse(strippedFilename);
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToBlobInfo.put(
|
||||
upperBoundTime, lazyExecutor.get().submit(() -> getBlobInfo(gcsBucket, filename)));
|
||||
upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
@@ -173,7 +180,7 @@ class GcsDiffFileLister {
|
||||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
} finally {
|
||||
lazyExecutor.get().shutdown();
|
||||
executor.shutdown();
|
||||
}
|
||||
|
||||
logger.atInfo().log(
|
||||
@@ -198,4 +205,36 @@ class GcsDiffFileLister {
|
||||
private BlobInfo getBlobInfo(String gcsBucket, String filename) {
|
||||
return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a prefix guaranteed to cover all commit log diff files in the given range.
|
||||
*
|
||||
* <p>The listObjects call can be fairly slow if we search over many thousands or tens of
|
||||
* thousands of files, so we restrict the search space. The commit logs have a file format of
|
||||
* "commit_diff_until_2021-05-11T06:48:00.070Z" so we can often filter down as far as the hour.
|
||||
*
|
||||
* <p>Here, we get the longest prefix possible based on which fields (year, month, day, hour) the
|
||||
* times in question have in common.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static String getCommitLogDiffPrefix(DateTime from, @Nullable DateTime to) {
|
||||
StringBuilder result = new StringBuilder(DIFF_FILE_PREFIX);
|
||||
if (to == null || from.getYear() != to.getYear()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(from.getYear()).append('-');
|
||||
if (from.getMonthOfYear() != to.getMonthOfYear()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02d-", from.getMonthOfYear()));
|
||||
if (from.getDayOfMonth() != to.getDayOfMonth()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02dT", from.getDayOfMonth()));
|
||||
if (from.getHourOfDay() != to.getHourOfDay()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02d:", from.getHourOfDay()));
|
||||
return result.toString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,13 +20,15 @@ import static google.registry.backup.RestoreCommitLogsAction.DRY_RUN_PARAM;
|
||||
import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.joda.time.Duration.standardHours;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.Key;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
@@ -120,26 +122,15 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
}
|
||||
try {
|
||||
logger.atInfo().log("Beginning replay of commit logs.");
|
||||
ImmutableList<BlobInfo> commitLogFiles = getFilesToReplay();
|
||||
String resultMessage;
|
||||
if (dryRun) {
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
ImmutableList<String> filenames =
|
||||
commitLogFiles.stream()
|
||||
.limit(10)
|
||||
.map(file -> file.getName())
|
||||
.collect(toImmutableList());
|
||||
String dryRunMessage =
|
||||
"Running in dry-run mode; would have processed %d files. They are (limit 10):\n"
|
||||
+ Joiner.on('\n').join(filenames);
|
||||
response.setPayload(dryRunMessage);
|
||||
logger.atInfo().log(dryRunMessage);
|
||||
resultMessage = executeDryRun();
|
||||
} else {
|
||||
replayFiles(commitLogFiles);
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
String message = "ReplayCommitLogsToSqlAction completed successfully.";
|
||||
response.setPayload(message);
|
||||
logger.atInfo().log(message);
|
||||
resultMessage = replayFiles();
|
||||
}
|
||||
response.setStatus(SC_OK);
|
||||
response.setPayload(resultMessage);
|
||||
logger.atInfo().log(resultMessage);
|
||||
} catch (Throwable t) {
|
||||
String message = "Errored out replaying files.";
|
||||
logger.atSevere().withCause(t).log(message);
|
||||
@@ -150,52 +141,78 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
private ImmutableList<BlobInfo> getFilesToReplay() {
|
||||
private String executeDryRun() {
|
||||
// Start at the first millisecond we haven't seen yet
|
||||
DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
logger.atInfo().log("Starting replay from: %s.", fromTime);
|
||||
// If there's an inconsistent file set, this will throw IllegalStateException and the job
|
||||
// will try later -- this is likely because an export hasn't finished yet.
|
||||
ImmutableList<BlobInfo> commitLogFiles =
|
||||
diffLister.listDiffFiles(gcsBucket, fromTime, /* current time */ null);
|
||||
logger.atInfo().log("Found %d new commit log files to process.", commitLogFiles.size());
|
||||
return commitLogFiles;
|
||||
DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
// Search through the end of the hour
|
||||
DateTime searchEndTime =
|
||||
searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
|
||||
ImmutableList<String> fileBatch =
|
||||
diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime).stream()
|
||||
.map(BlobInfo::getName)
|
||||
.collect(toImmutableList());
|
||||
return String.format(
|
||||
"Running in dry-run mode, the first set of commit log files processed would be from "
|
||||
+ "searching from %s to %s and would contain %d file(s). They are (limit 10): \n%s",
|
||||
searchStartTime,
|
||||
searchEndTime,
|
||||
fileBatch.size(),
|
||||
fileBatch.stream().limit(10).collect(toImmutableList()));
|
||||
}
|
||||
|
||||
private void replayFiles(ImmutableList<BlobInfo> commitLogFiles) {
|
||||
private String replayFiles() {
|
||||
DateTime replayTimeoutTime = clock.nowUtc().plus(REPLAY_TIMEOUT_DURATION);
|
||||
int processedFiles = 0;
|
||||
for (BlobInfo metadata : commitLogFiles) {
|
||||
// One transaction per GCS file
|
||||
jpaTm().transact(() -> processFile(metadata));
|
||||
processedFiles++;
|
||||
if (clock.nowUtc().isAfter(replayTimeoutTime)) {
|
||||
logger.atInfo().log(
|
||||
"Reached max execution time after replaying %d files, leaving %d files for next run.",
|
||||
processedFiles, commitLogFiles.size() - processedFiles);
|
||||
return;
|
||||
DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
int filesProcessed = 0;
|
||||
// Starting from one millisecond after the last file we processed, search for and import files
|
||||
// one hour at a time until we catch up to the current time or we hit the replay timeout (in
|
||||
// which case the next run will pick up from where we leave off).
|
||||
//
|
||||
// We use hour-long batches because GCS supports filename prefix-based searches.
|
||||
while (true) {
|
||||
if (isAtOrAfter(clock.nowUtc(), replayTimeoutTime)) {
|
||||
return String.format(
|
||||
"Reached max execution time after replaying %d file(s).", filesProcessed);
|
||||
}
|
||||
if (isBeforeOrAt(clock.nowUtc(), searchStartTime)) {
|
||||
return String.format(
|
||||
"Caught up to current time after replaying %d file(s).", filesProcessed);
|
||||
}
|
||||
// Search through the end of the hour
|
||||
DateTime searchEndTime =
|
||||
searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
|
||||
ImmutableList<BlobInfo> fileBatch =
|
||||
diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime);
|
||||
if (fileBatch.isEmpty()) {
|
||||
logger.atInfo().log(
|
||||
"No remaining files found in hour %s, continuing search in the next hour.",
|
||||
searchStartTime.toString("yyyy-MM-dd HH"));
|
||||
}
|
||||
for (BlobInfo file : fileBatch) {
|
||||
processFile(file);
|
||||
filesProcessed++;
|
||||
if (clock.nowUtc().isAfter(replayTimeoutTime)) {
|
||||
return String.format(
|
||||
"Reached max execution time after replaying %d file(s).", filesProcessed);
|
||||
}
|
||||
}
|
||||
searchStartTime = searchEndTime.plusMillis(1);
|
||||
}
|
||||
logger.atInfo().log("Replayed %d commit log files to SQL successfully.", processedFiles);
|
||||
}
|
||||
|
||||
private void processFile(BlobInfo metadata) {
|
||||
logger.atInfo().log(
|
||||
"Processing commit log file %s of size %d B.", metadata.getName(), metadata.getSize());
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
// Load and process the Datastore transactions one at a time
|
||||
ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
|
||||
CommitLogImports.loadEntitiesByTransaction(input);
|
||||
logger.atInfo().log(
|
||||
"Replaying %d transactions from commit log file %s.",
|
||||
allTransactions.size(), metadata.getName());
|
||||
allTransactions.forEach(this::replayTransaction);
|
||||
allTransactions.forEach(
|
||||
transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
|
||||
// if we succeeded, set the last-seen time
|
||||
DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
|
||||
SqlReplayCheckpoint.set(checkpoint);
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(checkpoint));
|
||||
logger.atInfo().log(
|
||||
"Replayed %d transactions from commit log file %s.",
|
||||
allTransactions.size(), metadata.getName());
|
||||
"Replayed %d transactions from commit log file %s with size %d B.",
|
||||
allTransactions.size(), metadata.getName(), metadata.getSize());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(
|
||||
"Errored out while replaying commit log file " + metadata.getName(), e);
|
||||
@@ -217,20 +234,25 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
|
||||
private void handleEntityPut(Entity entity) {
|
||||
Object ofyPojo = auditedOfy().toPojo(entity);
|
||||
if (ofyPojo instanceof DatastoreEntity) {
|
||||
DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
|
||||
datastoreEntity
|
||||
.toSqlEntity()
|
||||
.ifPresent(
|
||||
sqlEntity -> {
|
||||
ReplaySpecializer.beforeSqlSave(sqlEntity);
|
||||
jpaTm().put(sqlEntity);
|
||||
});
|
||||
} else {
|
||||
// this should never happen, but we shouldn't fail on it
|
||||
logger.atSevere().log(
|
||||
"%s does not implement DatastoreEntity, which is necessary for SQL replay.",
|
||||
ofyPojo.getClass());
|
||||
try {
|
||||
if (ofyPojo instanceof DatastoreEntity) {
|
||||
DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
|
||||
datastoreEntity
|
||||
.toSqlEntity()
|
||||
.ifPresent(
|
||||
sqlEntity -> {
|
||||
sqlEntity.beforeSqlSaveOnReplay();
|
||||
jpaTm().put(sqlEntity);
|
||||
});
|
||||
} else {
|
||||
// this should never happen, but we shouldn't fail on it
|
||||
logger.atSevere().log(
|
||||
"%s does not implement DatastoreEntity, which is necessary for SQL replay.",
|
||||
ofyPojo.getClass());
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().log("Error when replaying object %s", ofyPojo);
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,13 +268,18 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
"Skipping SQL delete for kind %s since it is not convertible.", key.getKind());
|
||||
return;
|
||||
}
|
||||
Class<?> entityClass = entityVKey.getKind();
|
||||
// Delete the key iff the class represents a JPA entity that is replicated
|
||||
if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
|
||||
&& !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
|
||||
&& entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
|
||||
ReplaySpecializer.beforeSqlDelete(entityVKey);
|
||||
jpaTm().delete(entityVKey);
|
||||
try {
|
||||
Class<?> entityClass = entityVKey.getKind();
|
||||
// Delete the key iff the class represents a JPA entity that is replicated
|
||||
if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
|
||||
&& !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
|
||||
&& entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
|
||||
ReplaySpecializer.beforeSqlDelete(entityVKey);
|
||||
jpaTm().delete(entityVKey);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().log("Error when deleting key %s", entityVKey);
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.persistence.PersistenceModule.SchemaManagerConnection;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
@@ -48,22 +48,18 @@ import javax.inject.Inject;
|
||||
public class WipeOutCloudSqlAction implements Runnable {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
// As a short-lived class, hardcode allowed projects here instead of using config files.
|
||||
private static final ImmutableSet<String> ALLOWED_PROJECTS =
|
||||
ImmutableSet.of("domain-registry-qa");
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
private final String projectId;
|
||||
private final Supplier<Connection> connectionSupplier;
|
||||
private final Response response;
|
||||
private final Retrier retrier;
|
||||
|
||||
@Inject
|
||||
WipeOutCloudSqlAction(
|
||||
@Config("projectId") String projectId,
|
||||
@SchemaManagerConnection Supplier<Connection> connectionSupplier,
|
||||
Response response,
|
||||
Retrier retrier) {
|
||||
this.projectId = projectId;
|
||||
this.connectionSupplier = connectionSupplier;
|
||||
this.response = response;
|
||||
this.retrier = retrier;
|
||||
@@ -73,9 +69,9 @@ public class WipeOutCloudSqlAction implements Runnable {
|
||||
public void run() {
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
|
||||
if (!ALLOWED_PROJECTS.contains(projectId)) {
|
||||
if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
|
||||
response.setStatus(SC_FORBIDDEN);
|
||||
response.setPayload("Wipeout is not allowed in " + projectId);
|
||||
response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -90,11 +86,11 @@ public class WipeOutCloudSqlAction implements Runnable {
|
||||
},
|
||||
e -> !(e instanceof SQLException));
|
||||
response.setStatus(SC_OK);
|
||||
response.setPayload("Wiped out Cloud SQL in " + projectId);
|
||||
response.setPayload("Wiped out Cloud SQL in " + RegistryEnvironment.get());
|
||||
} catch (RuntimeException e) {
|
||||
logger.atSevere().withCause(e).log("Failed to wipe out Cloud SQL data.");
|
||||
response.setStatus(SC_INTERNAL_SERVER_ERROR);
|
||||
response.setPayload("Failed to wipe out Cloud SQL in " + projectId);
|
||||
response.setPayload("Failed to wipe out Cloud SQL in " + RegistryEnvironment.get());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
@@ -49,9 +50,8 @@ public class WipeoutDatastoreAction implements Runnable {
|
||||
|
||||
private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";
|
||||
|
||||
// As a short-lived class, hardcode allowed projects here instead of using config files.
|
||||
private static final ImmutableSet<String> ALLOWED_PROJECTS =
|
||||
ImmutableSet.of("domain-registry-qa");
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
private final String projectId;
|
||||
private final String jobRegion;
|
||||
@@ -80,9 +80,9 @@ public class WipeoutDatastoreAction implements Runnable {
|
||||
public void run() {
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
|
||||
if (!ALLOWED_PROJECTS.contains(projectId)) {
|
||||
if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
|
||||
response.setStatus(SC_FORBIDDEN);
|
||||
response.setPayload("Wipeout is not allowed in " + projectId);
|
||||
response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.initsql;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
|
||||
import com.google.api.services.cloudkms.v1.model.DecryptRequest;
|
||||
import com.google.common.base.Strings;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.keyring.kms.KmsConnection;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Base64;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Decrypts data using Cloud KMS, with the same crypto key with which Cloud SQL credential files on
|
||||
* GCS was encrypted. See {@link BackupPaths#getCloudSQLCredentialFilePatterns} for more
|
||||
* information.
|
||||
*/
|
||||
public class CloudSqlCredentialDecryptor {
|
||||
|
||||
private static final String CRYPTO_KEY_NAME = "nomulus-tool-key";
|
||||
private final KmsConnection kmsConnection;
|
||||
|
||||
@Inject
|
||||
CloudSqlCredentialDecryptor(@Config("beamKmsConnection") KmsConnection kmsConnection) {
|
||||
this.kmsConnection = kmsConnection;
|
||||
}
|
||||
|
||||
public String decrypt(String data) {
|
||||
checkArgument(!Strings.isNullOrEmpty(data), "Null or empty data.");
|
||||
byte[] ciphertext = Base64.getDecoder().decode(data);
|
||||
// Re-encode for Cloud KMS JSON REST API, invoked through kmsConnection.
|
||||
String urlSafeCipherText = new DecryptRequest().encodeCiphertext(ciphertext).getCiphertext();
|
||||
return new String(
|
||||
kmsConnection.decrypt(CRYPTO_KEY_NAME, urlSafeCipherText), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.initsql;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Information needed to connect to a database, including JDBC URL, user name, password, and in the
|
||||
* case of Cloud SQL, the database instance's name.
|
||||
*/
|
||||
@AutoValue
|
||||
abstract class SqlAccessInfo {
|
||||
|
||||
abstract String jdbcUrl();
|
||||
|
||||
abstract String user();
|
||||
|
||||
abstract String password();
|
||||
|
||||
abstract Optional<String> cloudSqlInstanceName();
|
||||
|
||||
public static SqlAccessInfo createCloudSqlAccessInfo(
|
||||
String sqlInstanceName, String username, String password) {
|
||||
return new AutoValue_SqlAccessInfo(
|
||||
"jdbc:postgresql://google/postgres", username, password, Optional.of(sqlInstanceName));
|
||||
}
|
||||
|
||||
public static SqlAccessInfo createLocalSqlAccessInfo(
|
||||
String jdbcUrl, String username, String password) {
|
||||
return new AutoValue_SqlAccessInfo(jdbcUrl, username, password, Optional.empty());
|
||||
}
|
||||
}
|
||||
@@ -14,18 +14,29 @@
|
||||
|
||||
package google.registry.beam.invoicing;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.beam.BeamUtils.getQueryFromFile;
|
||||
import static org.apache.beam.sdk.values.TypeDescriptors.strings;
|
||||
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.beam.common.RegistryJpaIO.Read;
|
||||
import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey;
|
||||
import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.reporting.billing.BillingModule;
|
||||
import google.registry.util.DateTimeUtils;
|
||||
import google.registry.util.DomainNameUtils;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.LocalTime;
|
||||
import java.time.YearMonth;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Pattern;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.PipelineResult;
|
||||
import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
@@ -58,6 +69,9 @@ public class InvoicingPipeline implements Serializable {
|
||||
private static final DateTimeFormatter TIMESTAMP_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS");
|
||||
|
||||
private static final Pattern SQL_COMMENT_REGEX =
|
||||
Pattern.compile("^\\s*--.*\\n", Pattern.MULTILINE);
|
||||
|
||||
private final InvoicingPipelineOptions options;
|
||||
|
||||
InvoicingPipeline(InvoicingPipelineOptions options) {
|
||||
@@ -71,21 +85,60 @@ public class InvoicingPipeline implements Serializable {
|
||||
}
|
||||
|
||||
void setupPipeline(Pipeline pipeline) {
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
PCollection<BillingEvent> billingEvents =
|
||||
pipeline.apply(
|
||||
"Read BillingEvents from Bigquery",
|
||||
BigQueryIO.read(BillingEvent::parseFromRecord)
|
||||
.fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
|
||||
.withCoder(SerializableCoder.of(BillingEvent.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
options.getDatabase().equals("DATASTORE")
|
||||
? readFromBigQuery(options, pipeline)
|
||||
: readFromCloudSql(options, pipeline);
|
||||
|
||||
saveInvoiceCsv(billingEvents, options);
|
||||
|
||||
saveDetailedCsv(billingEvents, options);
|
||||
}
|
||||
|
||||
static PCollection<BillingEvent> readFromBigQuery(
|
||||
InvoicingPipelineOptions options, Pipeline pipeline) {
|
||||
return pipeline.apply(
|
||||
"Read BillingEvents from Bigquery",
|
||||
BigQueryIO.read(BillingEvent::parseFromRecord)
|
||||
.fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
|
||||
.withCoder(SerializableCoder.of(BillingEvent.class))
|
||||
.usingStandardSql()
|
||||
.withoutValidation()
|
||||
.withTemplateCompatibility());
|
||||
}
|
||||
|
||||
static PCollection<BillingEvent> readFromCloudSql(
|
||||
InvoicingPipelineOptions options, Pipeline pipeline) {
|
||||
Read<Object[], BillingEvent> read =
|
||||
RegistryJpaIO.read(
|
||||
makeCloudSqlQuery(options.getYearMonth()), false, InvoicingPipeline::parseRow);
|
||||
|
||||
return pipeline.apply("Read BillingEvents from Cloud SQL", read);
|
||||
}
|
||||
|
||||
private static BillingEvent parseRow(Object[] row) {
|
||||
google.registry.model.billing.BillingEvent.OneTime oneTime =
|
||||
(google.registry.model.billing.BillingEvent.OneTime) row[0];
|
||||
Registrar registrar = (Registrar) row[1];
|
||||
return BillingEvent.create(
|
||||
oneTime.getId(),
|
||||
DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
|
||||
DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
|
||||
registrar.getClientId(),
|
||||
registrar.getBillingIdentifier().toString(),
|
||||
registrar.getPoNumber().orElse(""),
|
||||
DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),
|
||||
oneTime.getReason().toString(),
|
||||
oneTime.getTargetId(),
|
||||
oneTime.getDomainRepoId(),
|
||||
Optional.ofNullable(oneTime.getPeriodYears()).orElse(0),
|
||||
oneTime.getCost().getCurrencyUnit().toString(),
|
||||
oneTime.getCost().getAmount().doubleValue(),
|
||||
String.join(
|
||||
" ", oneTime.getFlags().stream().map(Flag::toString).collect(toImmutableSet())));
|
||||
}
|
||||
|
||||
/** Transform that converts a {@code BillingEvent} into an invoice CSV row. */
|
||||
private static class GenerateInvoiceRows
|
||||
extends PTransform<PCollection<BillingEvent>, PCollection<String>> {
|
||||
@@ -171,6 +224,21 @@ public class InvoicingPipeline implements Serializable {
|
||||
.build();
|
||||
}
|
||||
|
||||
/** Create the Cloud SQL query for a given yearMonth at runtime. */
|
||||
static String makeCloudSqlQuery(String yearMonth) {
|
||||
YearMonth endMonth = YearMonth.parse(yearMonth).plusMonths(1);
|
||||
String queryWithComments =
|
||||
SqlTemplate.create(
|
||||
getQueryFromFile(InvoicingPipeline.class, "cloud_sql_billing_events.sql"))
|
||||
.put("FIRST_TIMESTAMP_OF_MONTH", yearMonth.concat("-01"))
|
||||
.put(
|
||||
"LAST_TIMESTAMP_OF_MONTH",
|
||||
String.format("%d-%d-01", endMonth.getYear(), endMonth.getMonthValue()))
|
||||
.build();
|
||||
// Remove the comments from the query string
|
||||
return SQL_COMMENT_REGEX.matcher(queryWithComments).replaceAll("");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
PipelineOptionsFactory.register(InvoicingPipelineOptions.class);
|
||||
InvoicingPipelineOptions options =
|
||||
|
||||
@@ -30,6 +30,11 @@ public interface InvoicingPipelineOptions extends RegistryPipelineOptions {
|
||||
|
||||
void setInvoiceFilePrefix(String value);
|
||||
|
||||
@Description("The database to read data from.")
|
||||
String getDatabase();
|
||||
|
||||
void setDatabase(String value);
|
||||
|
||||
@Description("The GCS bucket URL for invoices and detailed reports to be uploaded.")
|
||||
String getBillingBucketUrl();
|
||||
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.rde;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.rde.RdeMode;
|
||||
import google.registry.model.rde.RdeNamingUtils;
|
||||
import google.registry.model.rde.RdeRevision;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.Ghostryde;
|
||||
import google.registry.rde.PendingDeposit;
|
||||
import google.registry.rde.RdeCounter;
|
||||
import google.registry.rde.RdeMarshaller;
|
||||
import google.registry.rde.RdeResourceType;
|
||||
import google.registry.rde.RdeUtil;
|
||||
import google.registry.tldconfig.idn.IdnTableEnum;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeader;
|
||||
import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
|
||||
import google.registry.xml.ValidationMode;
|
||||
import google.registry.xml.XmlException;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.security.Security;
|
||||
import java.util.Optional;
|
||||
import org.apache.beam.sdk.options.PipelineOptions;
|
||||
import org.apache.beam.sdk.transforms.DoFn;
|
||||
import org.apache.beam.sdk.transforms.PTransform;
|
||||
import org.apache.beam.sdk.transforms.ParDo;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.apache.beam.sdk.values.PDone;
|
||||
import org.bouncycastle.jce.provider.BouncyCastleProvider;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
public class RdeIO {
|
||||
|
||||
@AutoValue
|
||||
abstract static class Write
|
||||
extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
|
||||
|
||||
abstract GcsUtils gcsUtils();
|
||||
|
||||
abstract String rdeBucket();
|
||||
|
||||
// It's OK to return a primitive array because we are only using it to construct the
|
||||
// PGPPublicKey, which is not serializable.
|
||||
@SuppressWarnings("mutable")
|
||||
abstract byte[] stagingKeyBytes();
|
||||
|
||||
abstract ValidationMode validationMode();
|
||||
|
||||
static Builder builder() {
|
||||
return new AutoValue_RdeIO_Write.Builder();
|
||||
}
|
||||
|
||||
@AutoValue.Builder
|
||||
abstract static class Builder {
|
||||
abstract Builder setGcsUtils(GcsUtils gcsUtils);
|
||||
|
||||
abstract Builder setRdeBucket(String value);
|
||||
|
||||
abstract Builder setStagingKeyBytes(byte[] value);
|
||||
|
||||
abstract Builder setValidationMode(ValidationMode value);
|
||||
|
||||
abstract Write build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PDone expand(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
|
||||
input
|
||||
.apply(
|
||||
"Write to GCS",
|
||||
ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
|
||||
.apply("Update cursors", ParDo.of(new CursorUpdater()));
|
||||
return PDone.in(input.getPipeline());
|
||||
}
|
||||
}
|
||||
|
||||
private static class RdeWriter
|
||||
extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private final GcsUtils gcsUtils;
|
||||
private final String rdeBucket;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final RdeMarshaller marshaller;
|
||||
|
||||
protected RdeWriter(
|
||||
GcsUtils gcsUtils,
|
||||
String rdeBucket,
|
||||
byte[] stagingKeyBytes,
|
||||
ValidationMode validationMode) {
|
||||
this.gcsUtils = gcsUtils;
|
||||
this.rdeBucket = rdeBucket;
|
||||
this.stagingKeyBytes = stagingKeyBytes;
|
||||
this.marshaller = new RdeMarshaller(validationMode);
|
||||
}
|
||||
|
||||
@Setup
|
||||
public void setup() {
|
||||
Security.addProvider(new BouncyCastleProvider());
|
||||
}
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(
|
||||
@Element KV<PendingDeposit, Iterable<DepositFragment>> kv,
|
||||
PipelineOptions options,
|
||||
OutputReceiver<KV<PendingDeposit, Integer>> outputReceiver) {
|
||||
PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
|
||||
PendingDeposit key = kv.getKey();
|
||||
Iterable<DepositFragment> fragments = kv.getValue();
|
||||
RdeCounter counter = new RdeCounter();
|
||||
|
||||
// Determine some basic things about the deposit.
|
||||
final RdeMode mode = key.mode();
|
||||
final String tld = key.tld();
|
||||
final DateTime watermark = key.watermark();
|
||||
final int revision =
|
||||
Optional.ofNullable(key.revision())
|
||||
.orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
|
||||
String id = RdeUtil.timestampToId(watermark);
|
||||
String prefix = options.getJobName();
|
||||
String basename = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
|
||||
if (key.manual()) {
|
||||
checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
|
||||
prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
|
||||
} else {
|
||||
prefix = prefix + "/" + basename;
|
||||
}
|
||||
BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
|
||||
// This file will contain the byte length (ASCII) of the raw unencrypted XML.
|
||||
//
|
||||
// This is necessary because RdeUploadAction creates a tar file which requires that the length
|
||||
// be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
|
||||
// length, so we just save it separately.
|
||||
BlobId xmlLengthFilename = BlobId.of(rdeBucket, prefix + ".xml.length");
|
||||
BlobId reportFilename = BlobId.of(rdeBucket, prefix + "-report.xml.ghostryde");
|
||||
|
||||
// These variables will be populated as we write the deposit XML and used for other files.
|
||||
boolean failed = false;
|
||||
XjcRdeHeader header;
|
||||
|
||||
// Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
|
||||
|
||||
logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
|
||||
Writer output = new OutputStreamWriter(ghostrydeEncoder, UTF_8)) {
|
||||
|
||||
// Output the top portion of the XML document.
|
||||
output.write(marshaller.makeHeader(id, watermark, RdeResourceType.getUris(mode), revision));
|
||||
|
||||
// Output XML fragments while counting them.
|
||||
for (DepositFragment fragment : fragments) {
|
||||
if (!fragment.xml().isEmpty()) {
|
||||
output.write(fragment.xml());
|
||||
counter.increment(fragment.type());
|
||||
}
|
||||
if (!fragment.error().isEmpty()) {
|
||||
failed = true;
|
||||
logger.atSevere().log("Fragment error: %s", fragment.error());
|
||||
}
|
||||
}
|
||||
|
||||
// Don't write the IDN elements for BRDA.
|
||||
if (mode == RdeMode.FULL) {
|
||||
for (IdnTableEnum idn : IdnTableEnum.values()) {
|
||||
output.write(marshaller.marshalIdn(idn.getTable()));
|
||||
counter.increment(RdeResourceType.IDN);
|
||||
}
|
||||
}
|
||||
|
||||
// Output XML that says how many resources were emitted.
|
||||
header = counter.makeHeader(tld, mode);
|
||||
output.write(marshaller.marshalOrDie(new XjcRdeHeaderElement(header)));
|
||||
|
||||
// Output the bottom of the XML document.
|
||||
output.write(marshaller.makeFooter());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
// If an entity was broken, abort after writing as much logs/deposit data as possible.
|
||||
verify(!failed, "RDE staging failed for TLD %s", tld);
|
||||
|
||||
// Write a tiny XML file to GCS containing some information about the deposit.
|
||||
//
|
||||
// This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
|
||||
if (mode == RdeMode.FULL) {
|
||||
logger.atInfo().log("Writing %s", reportFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
|
||||
counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
|
||||
} catch (IOException | XmlException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
// Now that we're done, output roll the cursor forward.
|
||||
if (key.manual()) {
|
||||
logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task");
|
||||
} else {
|
||||
outputReceiver.output(KV.of(key, revision));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(@Element KV<PendingDeposit, Integer> input) {
|
||||
tm().transact(
|
||||
() -> {
|
||||
PendingDeposit key = input.getKey();
|
||||
int revision = input.getValue();
|
||||
Registry registry = Registry.get(key.tld());
|
||||
Optional<Cursor> cursor =
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
tm().loadByKeyIfPresent(
|
||||
Cursor.createVKey(key.cursor(), registry.getTldStr())));
|
||||
DateTime position = getCursorTimeOrStartOfTime(cursor);
|
||||
checkState(key.interval() != null, "Interval must be present");
|
||||
DateTime newPosition = key.watermark().plus(key.interval());
|
||||
if (!position.isBefore(newPosition)) {
|
||||
logger.atWarning().log("Cursor has already been rolled forward.");
|
||||
return;
|
||||
}
|
||||
verify(
|
||||
position.equals(key.watermark()),
|
||||
"Partial ordering of RDE deposits broken: %s %s",
|
||||
position,
|
||||
key);
|
||||
tm().put(Cursor.create(key.cursor(), newPosition, registry));
|
||||
logger.atInfo().log(
|
||||
"Rolled forward %s on %s cursor to %s", key.cursor(), key.tld(), newPosition);
|
||||
RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,14 +18,18 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import dagger.BindsInstance;
|
||||
import dagger.Component;
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.config.CredentialModule;
|
||||
import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
@@ -33,6 +37,7 @@ import google.registry.model.host.HostResource;
|
||||
import google.registry.model.rde.RdeMode;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.Registrar.Type;
|
||||
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.PendingDeposit;
|
||||
@@ -50,6 +55,8 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Singleton;
|
||||
import javax.persistence.Entity;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.PipelineResult;
|
||||
@@ -58,6 +65,7 @@ import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.transforms.FlatMapElements;
|
||||
import org.apache.beam.sdk.transforms.Flatten;
|
||||
import org.apache.beam.sdk.transforms.GroupByKey;
|
||||
import org.apache.beam.sdk.transforms.Reshuffle;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
@@ -76,10 +84,15 @@ import org.joda.time.DateTime;
|
||||
* @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
|
||||
* Flex Templates</a>
|
||||
*/
|
||||
@Singleton
|
||||
public class RdePipeline implements Serializable {
|
||||
|
||||
private final RdeMarshaller marshaller;
|
||||
private final transient RdePipelineOptions options;
|
||||
private final ValidationMode mode;
|
||||
private final ImmutableSetMultimap<String, PendingDeposit> pendings;
|
||||
private final String rdeBucket;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
// Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
|
||||
// if sneaks into production we would get an extra signal.
|
||||
@@ -97,31 +110,43 @@ public class RdePipeline implements Serializable {
|
||||
+ (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
|
||||
}
|
||||
|
||||
RdePipeline(RdePipelineOptions options) throws IOException, ClassNotFoundException {
|
||||
this.marshaller = new RdeMarshaller(ValidationMode.valueOf(options.getValidationMode()));
|
||||
@Inject
|
||||
RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
|
||||
this.options = options;
|
||||
this.mode = ValidationMode.valueOf(options.getValidationMode());
|
||||
this.pendings = decodePendings(options.getPendings());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
PipelineResult run(Pipeline pipeline) {
|
||||
createFragments(pipeline);
|
||||
return pipeline.run();
|
||||
this.rdeBucket = options.getGcsBucket();
|
||||
this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
PipelineResult run() {
|
||||
return run(Pipeline.create());
|
||||
Pipeline pipeline = Pipeline.create(options);
|
||||
PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> fragments =
|
||||
createFragments(pipeline);
|
||||
persistData(fragments);
|
||||
return pipeline.run();
|
||||
}
|
||||
|
||||
PCollection<KV<PendingDeposit, DepositFragment>> createFragments(Pipeline pipeline) {
|
||||
PCollection<KV<PendingDeposit, DepositFragment>> fragments =
|
||||
PCollectionList.of(processRegistrars(pipeline))
|
||||
.and(processNonRegistrarEntities(pipeline, DomainBase.class))
|
||||
.and(processNonRegistrarEntities(pipeline, ContactResource.class))
|
||||
.and(processNonRegistrarEntities(pipeline, HostResource.class))
|
||||
.apply(Flatten.pCollections())
|
||||
.setCoder(
|
||||
KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)));
|
||||
return fragments;
|
||||
PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
|
||||
return PCollectionList.of(processRegistrars(pipeline))
|
||||
.and(processNonRegistrarEntities(pipeline, DomainBase.class))
|
||||
.and(processNonRegistrarEntities(pipeline, ContactResource.class))
|
||||
.and(processNonRegistrarEntities(pipeline, HostResource.class))
|
||||
.apply(Flatten.pCollections())
|
||||
.setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
|
||||
.apply("Group by PendingDeposit", GroupByKey.create());
|
||||
}
|
||||
|
||||
void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
|
||||
input.apply(
|
||||
"Write to GCS and update cursors",
|
||||
RdeIO.Write.builder()
|
||||
.setRdeBucket(rdeBucket)
|
||||
.setGcsUtils(gcsUtils)
|
||||
.setValidationMode(mode)
|
||||
.setStagingKeyBytes(stagingKeyBytes)
|
||||
.build());
|
||||
}
|
||||
|
||||
PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
|
||||
@@ -144,7 +169,8 @@ public class RdePipeline implements Serializable {
|
||||
.via(
|
||||
(VKey<Registrar> key) -> {
|
||||
Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
|
||||
DepositFragment fragment = marshaller.marshalRegistrar(registrar);
|
||||
DepositFragment fragment =
|
||||
new RdeMarshaller(mode).marshalRegistrar(registrar);
|
||||
return pendings.values().stream()
|
||||
.map(pending -> KV.of(pending, fragment))
|
||||
.collect(toImmutableSet());
|
||||
@@ -205,7 +231,8 @@ public class RdePipeline implements Serializable {
|
||||
Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
|
||||
// Convert resource to an XML fragment for each watermark/mode pair lazily and cache
|
||||
// the result.
|
||||
RdeFragmenter fragmenter = new RdeFragmenter(resourceAtTimes, marshaller);
|
||||
RdeFragmenter fragmenter =
|
||||
new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
|
||||
List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
|
||||
for (String tld : tlds) {
|
||||
for (PendingDeposit pending : pendings.get(tld)) {
|
||||
@@ -228,13 +255,14 @@ public class RdePipeline implements Serializable {
|
||||
* the original TLD to pending deposit map.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending)
|
||||
throws IOException, ClassNotFoundException {
|
||||
static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
|
||||
try (ObjectInputStream ois =
|
||||
new ObjectInputStream(
|
||||
new ByteArrayInputStream(
|
||||
BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
|
||||
return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
|
||||
} catch (IOException | ClassNotFoundException e) {
|
||||
throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,7 +270,7 @@ public class RdePipeline implements Serializable {
|
||||
* Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
|
||||
* worker by the pipeline launcher as a pipeline option.
|
||||
*/
|
||||
public static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
|
||||
static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
|
||||
throws IOException {
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
ObjectOutputStream oos = new ObjectOutputStream(baos);
|
||||
@@ -255,6 +283,21 @@ public class RdePipeline implements Serializable {
|
||||
public static void main(String[] args) throws IOException, ClassNotFoundException {
|
||||
PipelineOptionsFactory.register(RdePipelineOptions.class);
|
||||
RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
|
||||
new RdePipeline(options).run();
|
||||
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
|
||||
DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
|
||||
}
|
||||
|
||||
@Singleton
|
||||
@Component(modules = {CredentialModule.class, ConfigModule.class})
|
||||
interface RdePipelineComponent {
|
||||
RdePipeline rdePipeline();
|
||||
|
||||
@Component.Builder
|
||||
interface Builder {
|
||||
@BindsInstance
|
||||
Builder options(RdePipelineOptions options);
|
||||
|
||||
RdePipelineComponent build();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,4 +29,14 @@ public interface RdePipelineOptions extends RegistryPipelineOptions {
|
||||
String getValidationMode();
|
||||
|
||||
void setValidationMode(String value);
|
||||
|
||||
@Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
|
||||
String getGcsBucket();
|
||||
|
||||
void setGcsBucket(String value);
|
||||
|
||||
@Description("The Base64-encoded PGP public key to encrypt the deposits.")
|
||||
String getStagingKey();
|
||||
|
||||
void setStagingKey(String value);
|
||||
}
|
||||
|
||||
@@ -1267,16 +1267,34 @@ public final class RegistryConfig {
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningDays")
|
||||
public static int provideDaysToExpiration(RegistryConfigSettings config) {
|
||||
public static int provideExpirationWarningDays(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningDays;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningIntervalDays")
|
||||
public static int provideExpirationWarningIntervalDays(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningIntervalDays;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("minimumRsaKeyLength")
|
||||
public static int provideMinimumRsaKeyLength(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.minimumRsaKeyLength;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningEmailBodyText")
|
||||
public static String provideExpirationWarningEmailBodyText(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningEmailBodyText;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("expirationWarningEmailSubjectText")
|
||||
public static String provideExpirationWarningEmailSubjectText(RegistryConfigSettings config) {
|
||||
return config.sslCertificateValidation.expirationWarningEmailSubjectText;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Config("allowedEcdsaCurves")
|
||||
public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
|
||||
@@ -1476,21 +1494,6 @@ public final class RegistryConfig {
|
||||
return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether to replicate cloud SQL transactions to datastore.
|
||||
*
|
||||
* <p>If true, all cloud SQL transactions will be persisted as TransactionEntity objects in the
|
||||
* Transaction table and replayed against datastore in a cron job.
|
||||
*/
|
||||
public static boolean getCloudSqlReplicateTransactions() {
|
||||
return CONFIG_SETTINGS.get().cloudSql.replicateTransactions;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void overrideCloudSqlReplicateTransactions(boolean replicateTransactions) {
|
||||
CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
|
||||
}
|
||||
|
||||
/** Returns the roid suffix to be used for the roids of all contacts and hosts. */
|
||||
public static String getContactAndHostRoidSuffix() {
|
||||
return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;
|
||||
|
||||
@@ -126,7 +126,6 @@ public class RegistryConfigSettings {
|
||||
// TODO(05012021): remove username field after it is removed from all yaml files.
|
||||
public String username;
|
||||
public String instanceConnectionName;
|
||||
public boolean replicateTransactions;
|
||||
}
|
||||
|
||||
/** Configuration for Apache Beam (Cloud Dataflow). */
|
||||
@@ -228,7 +227,10 @@ public class RegistryConfigSettings {
|
||||
public static class SslCertificateValidation {
|
||||
public Map<String, Integer> maxValidityDaysSchedule;
|
||||
public int expirationWarningDays;
|
||||
public int expirationWarningIntervalDays;
|
||||
public int minimumRsaKeyLength;
|
||||
public Set<String> allowedEcdsaCurves;
|
||||
public String expirationWarningEmailBodyText;
|
||||
public String expirationWarningEmailSubjectText;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,9 +227,6 @@ cloudSql:
|
||||
jdbcUrl: jdbc:postgresql://localhost
|
||||
# This name is used by Cloud SQL when connecting to the database.
|
||||
instanceConnectionName: project-id:region:instance-id
|
||||
# Set this to true to replicate cloud SQL transactions to datastore in the
|
||||
# background.
|
||||
replicateTransactions: false
|
||||
|
||||
cloudDns:
|
||||
# Set both properties to null in Production.
|
||||
@@ -452,6 +449,15 @@ sslCertificateValidation:
|
||||
# The number of days before a certificate expires that indicates the
|
||||
# certificate is nearing expiration and warnings should be sent.
|
||||
expirationWarningDays: 30
|
||||
# The minimum number of days between two successive expiring notification emails.
|
||||
expirationWarningIntervalDays: 15
|
||||
# Text for expiring certificate notification email subject.
|
||||
expirationWarningEmailSubjectText: Certificate Expring Within 30 Days.
|
||||
# Text for expiring certificate notification email body that accepts 3 parameters:
|
||||
# registrar name, certificate type, and expiration date, respectively.
|
||||
expirationWarningEmailBodyText: |
|
||||
Hello Registrar %s,
|
||||
The %s certificate is expiring on %s.
|
||||
# The minimum number of bits an RSA key must contain.
|
||||
minimumRsaKeyLength: 2048
|
||||
# The ECDSA curves that are allowed for public keys.
|
||||
|
||||
@@ -24,6 +24,7 @@ import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.DateTimeUtils;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.StringWriter;
|
||||
import java.security.PublicKey;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.security.cert.CertificateFactory;
|
||||
@@ -32,10 +33,15 @@ import java.security.interfaces.ECPublicKey;
|
||||
import java.security.interfaces.RSAPublicKey;
|
||||
import java.util.Date;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import org.bouncycastle.jcajce.provider.asymmetric.util.EC5Util;
|
||||
import org.bouncycastle.jce.ECNamedCurveTable;
|
||||
import org.bouncycastle.jce.spec.ECNamedCurveParameterSpec;
|
||||
import org.bouncycastle.jce.spec.ECParameterSpec;
|
||||
import org.bouncycastle.openssl.jcajce.JcaMiscPEMGenerator;
|
||||
import org.bouncycastle.util.io.pem.PemObjectGenerator;
|
||||
import org.bouncycastle.util.io.pem.PemWriter;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Days;
|
||||
|
||||
@@ -43,10 +49,11 @@ import org.joda.time.Days;
|
||||
public class CertificateChecker {
|
||||
|
||||
private final ImmutableSortedMap<DateTime, Integer> maxValidityLengthSchedule;
|
||||
private final int daysToExpiration;
|
||||
private final int expirationWarningDays;
|
||||
private final int minimumRsaKeyLength;
|
||||
private final Clock clock;
|
||||
private final ImmutableSet<String> allowedEcdsaCurves;
|
||||
private final int expirationWarningIntervalDays;
|
||||
|
||||
/**
|
||||
* Constructs a CertificateChecker instance with the specified configuration parameters.
|
||||
@@ -72,6 +79,7 @@ public class CertificateChecker {
|
||||
@Config("maxValidityDaysSchedule")
|
||||
ImmutableSortedMap<DateTime, Integer> maxValidityDaysSchedule,
|
||||
@Config("expirationWarningDays") int expirationWarningDays,
|
||||
@Config("expirationWarningIntervalDays") int expirationWarningIntervalDays,
|
||||
@Config("minimumRsaKeyLength") int minimumRsaKeyLength,
|
||||
@Config("allowedEcdsaCurves") ImmutableSet<String> allowedEcdsaCurves,
|
||||
Clock clock) {
|
||||
@@ -79,12 +87,46 @@ public class CertificateChecker {
|
||||
maxValidityDaysSchedule.containsKey(START_OF_TIME),
|
||||
"Max validity length schedule must contain an entry for START_OF_TIME");
|
||||
this.maxValidityLengthSchedule = maxValidityDaysSchedule;
|
||||
this.daysToExpiration = expirationWarningDays;
|
||||
this.expirationWarningDays = expirationWarningDays;
|
||||
this.minimumRsaKeyLength = minimumRsaKeyLength;
|
||||
this.allowedEcdsaCurves = allowedEcdsaCurves;
|
||||
this.expirationWarningIntervalDays = expirationWarningIntervalDays;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
private static int getValidityLengthInDays(X509Certificate certificate) {
|
||||
DateTime start = DateTime.parse(certificate.getNotBefore().toInstant().toString());
|
||||
DateTime end = DateTime.parse(certificate.getNotAfter().toInstant().toString());
|
||||
return Days.daysBetween(start.withTimeAtStartOfDay(), end.withTimeAtStartOfDay()).getDays();
|
||||
}
|
||||
|
||||
/** Checks if the curve used for a public key is in the list of acceptable curves. */
|
||||
private static boolean checkCurveName(PublicKey key, ImmutableSet<String> allowedEcdsaCurves) {
|
||||
ECParameterSpec params;
|
||||
// These 2 different instances of PublicKey need to be handled separately since their OIDs are
|
||||
// encoded differently. More details on this can be found at
|
||||
// https://stackoverflow.com/questions/49895713/how-to-find-the-matching-curve-name-from-an-ecpublickey.
|
||||
if (key instanceof ECPublicKey) {
|
||||
ECPublicKey ecKey = (ECPublicKey) key;
|
||||
params = EC5Util.convertSpec(ecKey.getParams(), false);
|
||||
} else if (key instanceof org.bouncycastle.jce.interfaces.ECPublicKey) {
|
||||
org.bouncycastle.jce.interfaces.ECPublicKey ecKey =
|
||||
(org.bouncycastle.jce.interfaces.ECPublicKey) key;
|
||||
params = ecKey.getParameters();
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unrecognized instance of PublicKey.");
|
||||
}
|
||||
return allowedEcdsaCurves.stream()
|
||||
.anyMatch(
|
||||
curve -> {
|
||||
ECNamedCurveParameterSpec cParams = ECNamedCurveTable.getParameterSpec(curve);
|
||||
return cParams.getN().equals(params.getN())
|
||||
&& cParams.getH().equals(params.getH())
|
||||
&& cParams.getCurve().equals(params.getCurve())
|
||||
&& cParams.getG().equals(params.getG());
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the given certificate string for violations and throws an exception if any violations
|
||||
* exist.
|
||||
@@ -156,67 +198,57 @@ public class CertificateChecker {
|
||||
* the violations the certificate has.
|
||||
*/
|
||||
public ImmutableSet<CertificateViolation> checkCertificate(String certificateString) {
|
||||
X509Certificate certificate;
|
||||
return checkCertificate(getCertificate(certificateString));
|
||||
}
|
||||
|
||||
/** Converts the given string to a certificate object. */
|
||||
public X509Certificate getCertificate(String certificateStr) {
|
||||
X509Certificate certificate;
|
||||
try {
|
||||
certificate =
|
||||
(X509Certificate)
|
||||
CertificateFactory.getInstance("X509")
|
||||
.generateCertificate(new ByteArrayInputStream(certificateString.getBytes(UTF_8)));
|
||||
.generateCertificate(new ByteArrayInputStream(certificateStr.getBytes(UTF_8)));
|
||||
} catch (CertificateException e) {
|
||||
throw new IllegalArgumentException("Unable to read given certificate.");
|
||||
throw new IllegalArgumentException(
|
||||
String.format("Unable to read given certificate %s", certificateStr), e);
|
||||
}
|
||||
|
||||
return checkCertificate(certificate);
|
||||
return certificate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether the certificate is nearing expiration.
|
||||
*
|
||||
* <p>Note that this is <i>all</i> that it checks. The certificate itself may well be expired or
|
||||
* not yet valid and this message will still return false. So you definitely want to pair a call
|
||||
* to this method with a call to {@link #checkCertificate} to determine other issues with the
|
||||
* certificate that may be occurring.
|
||||
*/
|
||||
public boolean isNearingExpiration(X509Certificate certificate) {
|
||||
Date nearingExpirationDate =
|
||||
DateTime.parse(certificate.getNotAfter().toInstant().toString())
|
||||
.minusDays(daysToExpiration)
|
||||
.toDate();
|
||||
return clock.nowUtc().toDate().after(nearingExpirationDate);
|
||||
}
|
||||
|
||||
private static int getValidityLengthInDays(X509Certificate certificate) {
|
||||
DateTime start = DateTime.parse(certificate.getNotBefore().toInstant().toString());
|
||||
DateTime end = DateTime.parse(certificate.getNotAfter().toInstant().toString());
|
||||
return Days.daysBetween(start.withTimeAtStartOfDay(), end.withTimeAtStartOfDay()).getDays();
|
||||
}
|
||||
|
||||
/** Checks if the curve used for a public key is in the list of acceptable curves. */
|
||||
private static boolean checkCurveName(PublicKey key, ImmutableSet<String> allowedEcdsaCurves) {
|
||||
org.bouncycastle.jce.spec.ECParameterSpec params;
|
||||
// These 2 different instances of PublicKey need to be handled separately since their OIDs are
|
||||
// encoded differently. More details on this can be found at
|
||||
// https://stackoverflow.com/questions/49895713/how-to-find-the-matching-curve-name-from-an-ecpublickey.
|
||||
if (key instanceof ECPublicKey) {
|
||||
ECPublicKey ecKey = (ECPublicKey) key;
|
||||
params = EC5Util.convertSpec(ecKey.getParams(), false);
|
||||
} else if (key instanceof org.bouncycastle.jce.interfaces.ECPublicKey) {
|
||||
org.bouncycastle.jce.interfaces.ECPublicKey ecKey =
|
||||
(org.bouncycastle.jce.interfaces.ECPublicKey) key;
|
||||
params = ecKey.getParameters();
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unrecognized instance of PublicKey.");
|
||||
/** Serializes the certificate object to a certificate string. */
|
||||
public String serializeCertificate(X509Certificate certificate) throws Exception {
|
||||
StringWriter sw = new StringWriter();
|
||||
try (PemWriter pw = new PemWriter(sw)) {
|
||||
PemObjectGenerator generator = new JcaMiscPEMGenerator(certificate);
|
||||
pw.writeObject(generator);
|
||||
}
|
||||
return allowedEcdsaCurves.stream()
|
||||
.anyMatch(
|
||||
curve -> {
|
||||
ECNamedCurveParameterSpec cParams = ECNamedCurveTable.getParameterSpec(curve);
|
||||
return cParams.getN().equals(params.getN())
|
||||
&& cParams.getH().equals(params.getH())
|
||||
&& cParams.getCurve().equals(params.getCurve())
|
||||
&& cParams.getG().equals(params.getG());
|
||||
});
|
||||
return sw.toString();
|
||||
}
|
||||
|
||||
/** Returns whether the client should receive a notification email. */
|
||||
public boolean shouldReceiveExpiringNotification(
|
||||
@Nullable DateTime lastExpiringNotificationSentDate, String certificateStr) {
|
||||
X509Certificate certificate = getCertificate(certificateStr);
|
||||
DateTime now = clock.nowUtc();
|
||||
// expiration date is one day after lastValidDate
|
||||
Date lastValidDate = certificate.getNotAfter();
|
||||
if (lastValidDate.before(now.toDate())) {
|
||||
return false;
|
||||
}
|
||||
/*
|
||||
* Client should receive a notification if :
|
||||
* 1) client has never received notification and the certificate has entered
|
||||
* the expiring period, OR
|
||||
* 2) client has received notification but the interval between now and
|
||||
* lastExpiringNotificationSentDate is greater than expirationWarningIntervalDays.
|
||||
*/
|
||||
return !lastValidDate.after(now.plusDays(expirationWarningDays).toDate())
|
||||
&& (lastExpiringNotificationSentDate == null
|
||||
|| !lastExpiringNotificationSentDate
|
||||
.plusDays(expirationWarningIntervalDays)
|
||||
.toDate()
|
||||
.after(now.toDate()));
|
||||
}
|
||||
|
||||
private String getViolationDisplayMessage(CertificateViolation certificateViolation) {
|
||||
|
||||
@@ -28,14 +28,8 @@ import org.bouncycastle.openpgp.PGPKeyPair;
|
||||
import org.bouncycastle.openpgp.PGPPrivateKey;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
|
||||
/**
|
||||
* A {@link Keyring} implementation which stores encrypted secrets in Datastore and decrypts them
|
||||
* using encryption keys stored in Cloud KMS.
|
||||
*
|
||||
* @see <a href="https://cloud.google.com/kms/docs/">Google Cloud Key Management Service
|
||||
* Documentation</a>
|
||||
*/
|
||||
// TODO(2021-07-01): rename this class to SecretManagerKeyring and delete KmsSecretRevision
|
||||
/** A {@link Keyring} implementation which stores sensitive data in the Secret Manager. */
|
||||
// TODO(2021-08-01): rename this class to SecretManagerKeyring and update config files.
|
||||
public class KmsKeyring implements Keyring {
|
||||
|
||||
/** Key labels for private key secrets. */
|
||||
|
||||
@@ -43,8 +43,6 @@ import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.label.PremiumList;
|
||||
import google.registry.model.registry.label.ReservedList;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.server.KmsSecretRevision;
|
||||
import google.registry.model.server.Lock;
|
||||
import google.registry.model.server.ServerSecret;
|
||||
import google.registry.model.tmch.ClaimsList;
|
||||
@@ -88,8 +86,6 @@ public final class EntityClasses {
|
||||
HistoryEntry.class,
|
||||
HostHistory.class,
|
||||
HostResource.class,
|
||||
KmsSecret.class,
|
||||
KmsSecretRevision.class,
|
||||
LastSqlTransaction.class,
|
||||
Lock.class,
|
||||
PollMessage.class,
|
||||
|
||||
@@ -225,7 +225,7 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton
|
||||
@VisibleForTesting
|
||||
static TimedTransitionProperty<MigrationState, MigrationStateTransition> getUncached() {
|
||||
return ofyTm()
|
||||
.transact(
|
||||
.transactNew(
|
||||
() ->
|
||||
ofyTm()
|
||||
.loadSingleton(DatabaseMigrationStateSchedule.class)
|
||||
|
||||
@@ -136,9 +136,11 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
|
||||
}
|
||||
|
||||
// Used to fill out the contactBase field during asynchronous replay
|
||||
public static void beforeSqlSave(ContactHistory contactHistory) {
|
||||
contactHistory.contactBase =
|
||||
jpaTm().loadByKey(VKey.createSql(ContactResource.class, contactHistory.getContactRepoId()));
|
||||
@Override
|
||||
public void beforeSqlSaveOnReplay() {
|
||||
if (contactBase == null) {
|
||||
contactBase = jpaTm().getEntityManager().find(ContactResource.class, getContactRepoId());
|
||||
}
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link ContactHistory} entity. */
|
||||
|
||||
@@ -294,9 +294,26 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
|
||||
}
|
||||
|
||||
// Used to fill out the domainContent field during asynchronous replay
|
||||
public static void beforeSqlSave(DomainHistory domainHistory) {
|
||||
domainHistory.domainContent =
|
||||
jpaTm().loadByKey(VKey.createSql(DomainBase.class, domainHistory.getDomainRepoId()));
|
||||
@Override
|
||||
public void beforeSqlSaveOnReplay() {
|
||||
if (domainContent == null) {
|
||||
domainContent = jpaTm().getEntityManager().find(DomainBase.class, getDomainRepoId());
|
||||
fillAuxiliaryFieldsFromDomain(this);
|
||||
}
|
||||
}
|
||||
|
||||
private static void fillAuxiliaryFieldsFromDomain(DomainHistory domainHistory) {
|
||||
if (domainHistory.domainContent != null) {
|
||||
domainHistory.nsHosts = nullToEmptyImmutableCopy(domainHistory.domainContent.nsHosts);
|
||||
domainHistory.dsDataHistories =
|
||||
nullToEmptyImmutableCopy(domainHistory.domainContent.getDsData()).stream()
|
||||
.map(dsData -> DomainDsDataHistory.createFrom(domainHistory.id, dsData))
|
||||
.collect(toImmutableSet());
|
||||
domainHistory.gracePeriodHistories =
|
||||
nullToEmptyImmutableCopy(domainHistory.domainContent.getGracePeriods()).stream()
|
||||
.map(gracePeriod -> GracePeriodHistory.createFrom(domainHistory.id, gracePeriod))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link DomainHistory} entity. */
|
||||
@@ -391,17 +408,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
|
||||
// Note that we cannot assert that instance.domainContent is not null here because this
|
||||
// builder is also used to convert legacy HistoryEntry objects to DomainHistory, when
|
||||
// domainContent is not available.
|
||||
if (instance.domainContent != null) {
|
||||
instance.nsHosts = nullToEmptyImmutableCopy(instance.domainContent.nsHosts);
|
||||
instance.dsDataHistories =
|
||||
nullToEmptyImmutableCopy(instance.domainContent.getDsData()).stream()
|
||||
.map(dsData -> DomainDsDataHistory.createFrom(instance.id, dsData))
|
||||
.collect(toImmutableSet());
|
||||
instance.gracePeriodHistories =
|
||||
nullToEmptyImmutableCopy(instance.domainContent.getGracePeriods()).stream()
|
||||
.map(gracePeriod -> GracePeriodHistory.createFrom(instance.id, gracePeriod))
|
||||
.collect(toImmutableSet());
|
||||
}
|
||||
fillAuxiliaryFieldsFromDomain(instance);
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,9 +136,11 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
|
||||
}
|
||||
|
||||
// Used to fill out the hostBase field during asynchronous replay
|
||||
public static void beforeSqlSave(HostHistory hostHistory) {
|
||||
hostHistory.hostBase =
|
||||
jpaTm().loadByKey(VKey.createSql(HostResource.class, hostHistory.getHostRepoId()));
|
||||
@Override
|
||||
public void beforeSqlSaveOnReplay() {
|
||||
if (hostBase == null) {
|
||||
hostBase = jpaTm().getEntityManager().find(HostResource.class, getHostRepoId());
|
||||
}
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link HostHistory} entity. */
|
||||
|
||||
@@ -154,7 +154,7 @@ public final class RdeRevision extends BackupGroupRoot implements NonReplicatedE
|
||||
}
|
||||
|
||||
/** Class to represent the composite primary key of {@link RdeRevision} entity. */
|
||||
static class RdeRevisionId extends ImmutableObject implements Serializable {
|
||||
public static class RdeRevisionId extends ImmutableObject implements Serializable {
|
||||
|
||||
String tld;
|
||||
|
||||
@@ -169,7 +169,7 @@ public final class RdeRevision extends BackupGroupRoot implements NonReplicatedE
|
||||
/** Hibernate requires this default constructor. */
|
||||
private RdeRevisionId() {}
|
||||
|
||||
static RdeRevisionId create(String tld, LocalDate date, RdeMode mode) {
|
||||
public static RdeRevisionId create(String tld, LocalDate date, RdeMode mode) {
|
||||
RdeRevisionId instance = new RdeRevisionId();
|
||||
instance.tld = tld;
|
||||
instance.date = date;
|
||||
|
||||
@@ -37,6 +37,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
|
||||
import static google.registry.util.CollectionUtils.nullToEmptyImmutableSortedCopy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.PasswordUtils.SALT_SUPPLIER;
|
||||
import static google.registry.util.PasswordUtils.hashPassword;
|
||||
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
@@ -458,6 +459,14 @@ public class Registrar extends ImmutableObject
|
||||
/** The time that the certificate was last updated. */
|
||||
DateTime lastCertificateUpdateTime;
|
||||
|
||||
/** The time that an expiring certificate notification email was sent to the registrar. */
|
||||
DateTime lastExpiringCertNotificationSentDate = START_OF_TIME;
|
||||
|
||||
/**
|
||||
* The time that an expiring failover certificate notification email was sent to the registrar.
|
||||
*/
|
||||
DateTime lastExpiringFailoverCertNotificationSentDate = START_OF_TIME;
|
||||
|
||||
/** Telephone support passcode (5-digit numeric) */
|
||||
String phonePasscode;
|
||||
|
||||
@@ -508,6 +517,14 @@ public class Registrar extends ImmutableObject
|
||||
return lastCertificateUpdateTime;
|
||||
}
|
||||
|
||||
public DateTime getLastExpiringCertNotificationSentDate() {
|
||||
return lastExpiringCertNotificationSentDate;
|
||||
}
|
||||
|
||||
public DateTime getLastExpiringFailoverCertNotificationSentDate() {
|
||||
return lastExpiringFailoverCertNotificationSentDate;
|
||||
}
|
||||
|
||||
public String getRegistrarName() {
|
||||
return registrarName;
|
||||
}
|
||||
@@ -671,6 +688,10 @@ public class Registrar extends ImmutableObject
|
||||
.putString("creationTime", creationTime.getTimestamp())
|
||||
.putString("lastUpdateTime", lastUpdateTime.getTimestamp())
|
||||
.putString("lastCertificateUpdateTime", lastCertificateUpdateTime)
|
||||
.putString("lastExpiringCertNotificationSentDate", lastExpiringCertNotificationSentDate)
|
||||
.putString(
|
||||
"lastExpiringFailoverCertNotificationSentDate",
|
||||
lastExpiringFailoverCertNotificationSentDate)
|
||||
.put("registrarName", registrarName)
|
||||
.put("type", type)
|
||||
.put("state", state)
|
||||
@@ -839,6 +860,19 @@ public class Registrar extends ImmutableObject
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setLastExpiringCertNotificationSentDate(DateTime now) {
|
||||
checkArgumentNotNull(now, "Registrar lastExpiringCertNotificationSentDate cannot be null");
|
||||
getInstance().lastExpiringCertNotificationSentDate = now;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setLastExpiringFailoverCertNotificationSentDate(DateTime now) {
|
||||
checkArgumentNotNull(
|
||||
now, "Registrar lastExpiringFailoverCertNotificationSentDate cannot be null");
|
||||
getInstance().lastExpiringFailoverCertNotificationSentDate = now;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setFailoverClientCertificate(String clientCertificate, DateTime now) {
|
||||
clientCertificate = emptyToNull(clientCertificate);
|
||||
String clientCertificateHash = calculateHash(clientCertificate);
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.InCrossTld;
|
||||
import google.registry.model.annotations.ReportedOn;
|
||||
import google.registry.model.common.EntityGroupRoot;
|
||||
import google.registry.schema.replay.DatastoreOnlyEntity;
|
||||
|
||||
/** Pointer to the latest {@link KmsSecretRevision}. */
|
||||
@Entity
|
||||
@ReportedOn
|
||||
@InCrossTld
|
||||
public class KmsSecret extends ImmutableObject implements DatastoreOnlyEntity {
|
||||
|
||||
/** The unique name of this {@link KmsSecret}. */
|
||||
@Id String name;
|
||||
|
||||
@Parent Key<EntityGroupRoot> parent = getCrossTldKey();
|
||||
|
||||
/** The pointer to the latest {@link KmsSecretRevision}. */
|
||||
Key<KmsSecretRevision> latestRevision;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public Key<KmsSecretRevision> getLatestRevision() {
|
||||
return latestRevision;
|
||||
}
|
||||
|
||||
public static KmsSecret create(String name, KmsSecretRevision latestRevision) {
|
||||
KmsSecret instance = new KmsSecret();
|
||||
instance.name = name;
|
||||
instance.latestRevision = Key.create(latestRevision);
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
|
||||
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import com.googlecode.objectify.annotation.Ignore;
|
||||
import com.googlecode.objectify.annotation.OnLoad;
|
||||
import com.googlecode.objectify.annotation.Parent;
|
||||
import google.registry.model.Buildable;
|
||||
import google.registry.model.CreateAutoTimestamp;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.InCrossTld;
|
||||
import google.registry.model.annotations.ReportedOn;
|
||||
import google.registry.schema.replay.NonReplicatedEntity;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Index;
|
||||
import javax.persistence.PostLoad;
|
||||
import javax.persistence.Table;
|
||||
import javax.persistence.Transient;
|
||||
|
||||
/**
|
||||
* An encrypted value.
|
||||
*
|
||||
* <p>Used to store passwords and other sensitive information in Datastore. Multiple versions of a
|
||||
* {@link KmsSecretRevision} may be persisted but only the latest version is primary. A key to the
|
||||
* primary version is stored by {@link KmsSecret#latestRevision}.
|
||||
*
|
||||
* <p>The value can be encrypted and decrypted using Cloud KMS.
|
||||
*
|
||||
* <p>Note that the primary key of this entity is {@link #revisionKey}, which is auto-generated by
|
||||
* the database. So, if a retry of insertion happens after the previous attempt unexpectedly
|
||||
* succeeds, we will end up with having two exact same revisions that differ only by revisionKey.
|
||||
* This is fine though, because we only use the revision with the highest revisionKey.
|
||||
*
|
||||
* <p>TODO(b/177567432): remove Datastore-specific fields post-Registry-3.0-migration and rename to
|
||||
* KmsSecret.
|
||||
*
|
||||
* @see <a href="https://cloud.google.com/kms/docs/">Google Cloud Key Management Service
|
||||
* Documentation</a>
|
||||
* @see google.registry.keyring.kms.KmsKeyring
|
||||
*/
|
||||
@Entity
|
||||
@ReportedOn
|
||||
@javax.persistence.Entity(name = "KmsSecret")
|
||||
@Table(indexes = {@Index(columnList = "secretName")})
|
||||
@InCrossTld
|
||||
public class KmsSecretRevision extends ImmutableObject implements NonReplicatedEntity {
|
||||
|
||||
/**
|
||||
* The maximum allowable secret size. Although Datastore allows entities up to 1 MB in size,
|
||||
* BigQuery imports of Datastore backups limit individual columns (entity attributes) to 64 KB.
|
||||
*/
|
||||
private static final int MAX_SECRET_SIZE_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* The revision of this secret.
|
||||
*
|
||||
* <p>TODO(b/177567432): change name of the variable to revisionId once we're off Datastore
|
||||
*/
|
||||
@Id
|
||||
@javax.persistence.Id
|
||||
@Column(name = "revisionId")
|
||||
long revisionKey;
|
||||
|
||||
/** The parent {@link KmsSecret} which contains metadata about this {@link KmsSecretRevision}. */
|
||||
@Parent @Transient Key<KmsSecret> parent;
|
||||
@Column(nullable = false)
|
||||
@Ignore
|
||||
String secretName;
|
||||
|
||||
/**
|
||||
* The name of the {@code cryptoKeyVersion} associated with this {@link KmsSecretRevision}.
|
||||
*
|
||||
* <p>TODO: change name of the variable to cryptoKeyVersionName once we're off Datastore
|
||||
*
|
||||
* @see <a
|
||||
* href="https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys.cryptoKeyVersions">projects.locations.keyRings.cryptoKeys.cryptoKeyVersions</a>
|
||||
*/
|
||||
@Column(nullable = false, name = "cryptoKeyVersionName")
|
||||
String kmsCryptoKeyVersionName;
|
||||
|
||||
/**
|
||||
* The base64-encoded encrypted value of this {@link KmsSecretRevision} as returned by the Cloud
|
||||
* KMS API.
|
||||
*
|
||||
* @see <a
|
||||
* href="https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys/encrypt">projects.locations.keyRings.cryptoKeys.encrypt</a>
|
||||
*/
|
||||
@Column(nullable = false)
|
||||
String encryptedValue;
|
||||
|
||||
/** An automatically managed creation timestamp. */
|
||||
@Column(nullable = false)
|
||||
CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
|
||||
|
||||
public String getKmsCryptoKeyVersionName() {
|
||||
return kmsCryptoKeyVersionName;
|
||||
}
|
||||
|
||||
public String getEncryptedValue() {
|
||||
return encryptedValue;
|
||||
}
|
||||
|
||||
// When loading from SQL, fill out the Datastore-specific field
|
||||
@PostLoad
|
||||
void postLoad() {
|
||||
parent = Key.create(getCrossTldKey(), KmsSecret.class, secretName);
|
||||
}
|
||||
|
||||
// When loading from Datastore, fill out the SQL-specific field
|
||||
@OnLoad
|
||||
void onLoad() {
|
||||
secretName = parent.getName();
|
||||
}
|
||||
|
||||
/** A builder for constructing {@link KmsSecretRevision} entities, since they are immutable. */
|
||||
public static class Builder extends Buildable.Builder<KmsSecretRevision> {
|
||||
|
||||
public Builder setKmsCryptoKeyVersionName(String kmsCryptoKeyVersionName) {
|
||||
getInstance().kmsCryptoKeyVersionName = kmsCryptoKeyVersionName;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEncryptedValue(String encryptedValue) {
|
||||
checkArgument(
|
||||
encryptedValue.length() <= MAX_SECRET_SIZE_BYTES,
|
||||
"Secret is greater than %s bytes",
|
||||
MAX_SECRET_SIZE_BYTES);
|
||||
|
||||
getInstance().encryptedValue = encryptedValue;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the parent {@link KmsSecret}.
|
||||
*
|
||||
* <p>The secret may not exist yet, so it is referred to by name rather than by reference.
|
||||
*/
|
||||
public Builder setParent(String secretName) {
|
||||
getInstance().parent = Key.create(getCrossTldKey(), KmsSecret.class, secretName);
|
||||
getInstance().secretName = secretName;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* A {@link KmsSecretRevision} DAO for Cloud SQL.
|
||||
*
|
||||
* <p>TODO(b/177567432): Rename this class to KmsSecretDao after migrating to Cloud SQL.
|
||||
*/
|
||||
public class KmsSecretRevisionSqlDao {
|
||||
|
||||
private KmsSecretRevisionSqlDao() {}
|
||||
|
||||
/** Saves the given KMS secret revision. */
|
||||
public static void save(KmsSecretRevision kmsSecretRevision) {
|
||||
checkArgumentNotNull(kmsSecretRevision, "kmsSecretRevision cannot be null");
|
||||
jpaTm().assertInTransaction();
|
||||
jpaTm().put(kmsSecretRevision);
|
||||
}
|
||||
|
||||
/** Returns the latest revision for the secret name given, or absent if nonexistent. */
|
||||
public static Optional<KmsSecretRevision> getLatestRevision(String secretName) {
|
||||
checkArgument(!isNullOrEmpty(secretName), "secretName cannot be null or empty");
|
||||
jpaTm().assertInTransaction();
|
||||
return jpaTm()
|
||||
.query(
|
||||
"FROM KmsSecret ks WHERE ks.revisionKey IN (SELECT MAX(revisionKey) FROM "
|
||||
+ "KmsSecret subKs WHERE subKs.secretName = :secretName)",
|
||||
KmsSecretRevision.class)
|
||||
.setParameter("secretName", secretName)
|
||||
.getResultStream()
|
||||
.findFirst();
|
||||
}
|
||||
}
|
||||
+25
-5
@@ -30,14 +30,14 @@ import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
|
||||
import google.registry.model.ofy.DatastoreTransactionManager;
|
||||
import google.registry.model.server.KmsSecret;
|
||||
import google.registry.model.tmch.ClaimsList.ClaimsListSingleton;
|
||||
import google.registry.persistence.JpaRetries;
|
||||
import google.registry.persistence.VKey;
|
||||
@@ -94,8 +94,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
EppResourceIndex.class,
|
||||
ForeignKeyContactIndex.class,
|
||||
ForeignKeyDomainIndex.class,
|
||||
ForeignKeyHostIndex.class,
|
||||
KmsSecret.class);
|
||||
ForeignKeyHostIndex.class);
|
||||
|
||||
// EntityManagerFactory is thread safe.
|
||||
private final EntityManagerFactory emf;
|
||||
@@ -105,6 +104,10 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
// synchronously.
|
||||
private final ThreadLocal<TransactionInfo> transactionInfo =
|
||||
ThreadLocal.withInitial(TransactionInfo::new);
|
||||
// If this value is present, use it to determine whether or not to replay SQL transactions to
|
||||
// Datastore, rather than using the schedule stored in Datastore.
|
||||
private static ThreadLocal<Optional<Boolean>> replaySqlToDatastoreOverrideForTest =
|
||||
ThreadLocal.withInitial(Optional::empty);
|
||||
|
||||
public JpaTransactionManagerImpl(EntityManagerFactory emf, Clock clock) {
|
||||
this.emf = emf;
|
||||
@@ -738,6 +741,16 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
return entity;
|
||||
}
|
||||
|
||||
/** Sets the override to always/never replay SQL transactions to Datastore. */
|
||||
public static void setReplaySqlToDatastoreOverrideForTest(boolean replaySqlToDs) {
|
||||
replaySqlToDatastoreOverrideForTest.set(Optional.of(replaySqlToDs));
|
||||
}
|
||||
|
||||
/** Removes the replay-SQL-to-Datastore override; the migration schedule will then be used. */
|
||||
public static void removeReplaySqlToDsOverrideForTest() {
|
||||
replaySqlToDatastoreOverrideForTest.set(Optional.empty());
|
||||
}
|
||||
|
||||
private static class TransactionInfo {
|
||||
EntityManager entityManager;
|
||||
boolean inTransaction = false;
|
||||
@@ -757,7 +770,14 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
|
||||
checkArgumentNotNull(clock);
|
||||
inTransaction = true;
|
||||
transactionTime = clock.nowUtc();
|
||||
if (withBackup && RegistryConfig.getCloudSqlReplicateTransactions()) {
|
||||
if (withBackup
|
||||
&& replaySqlToDatastoreOverrideForTest
|
||||
.get()
|
||||
.orElseGet(
|
||||
() ->
|
||||
DatabaseMigrationStateSchedule.getValueAtTime(transactionTime)
|
||||
.getReplayDirection()
|
||||
.equals(ReplayDirection.SQL_TO_DATASTORE))) {
|
||||
contentsBuilder = new Transaction.Builder();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ public abstract class PendingDeposit implements Serializable {
|
||||
return new AutoValue_PendingDeposit(false, tld, watermark, mode, cursor, interval, null, null);
|
||||
}
|
||||
|
||||
static PendingDeposit createInManualOperation(
|
||||
public static PendingDeposit createInManualOperation(
|
||||
String tld,
|
||||
DateTime watermark,
|
||||
RdeMode mode,
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
package google.registry.reporting.billing;
|
||||
|
||||
import static google.registry.beam.BeamUtils.createJobName;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.reporting.ReportingModule.DATABASE;
|
||||
import static google.registry.reporting.ReportingUtils.enqueueBeamReportingTask;
|
||||
import static google.registry.reporting.billing.BillingModule.PARAM_SHOULD_PUBLISH;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
@@ -70,6 +72,7 @@ public class GenerateInvoicesAction implements Runnable {
|
||||
private final Clock clock;
|
||||
private final Response response;
|
||||
private final Dataflow dataflow;
|
||||
private final String database;
|
||||
|
||||
@Inject
|
||||
GenerateInvoicesAction(
|
||||
@@ -79,6 +82,7 @@ public class GenerateInvoicesAction implements Runnable {
|
||||
@Config("billingBucketUrl") String billingBucketUrl,
|
||||
@Config("invoiceFilePrefix") String invoiceFilePrefix,
|
||||
@Parameter(PARAM_SHOULD_PUBLISH) boolean shouldPublish,
|
||||
@Parameter(DATABASE) String database,
|
||||
YearMonth yearMonth,
|
||||
BillingEmailUtils emailUtils,
|
||||
Clock clock,
|
||||
@@ -87,9 +91,15 @@ public class GenerateInvoicesAction implements Runnable {
|
||||
this.projectId = projectId;
|
||||
this.jobRegion = jobRegion;
|
||||
this.stagingBucketUrl = stagingBucketUrl;
|
||||
// When generating the invoices using Cloud SQL before database cutover, save the reports in a
|
||||
// separate bucket so that it does not overwrite the Datastore invoices.
|
||||
if (tm().isOfy() && database.equals("CLOUD_SQL")) {
|
||||
billingBucketUrl = billingBucketUrl.concat("-sql");
|
||||
}
|
||||
this.billingBucketUrl = billingBucketUrl;
|
||||
this.invoiceFilePrefix = invoiceFilePrefix;
|
||||
this.shouldPublish = shouldPublish;
|
||||
this.database = database;
|
||||
this.yearMonth = yearMonth;
|
||||
this.emailUtils = emailUtils;
|
||||
this.clock = clock;
|
||||
@@ -113,6 +123,8 @@ public class GenerateInvoicesAction implements Runnable {
|
||||
yearMonth.toString("yyyy-MM"),
|
||||
"invoiceFilePrefix",
|
||||
invoiceFilePrefix,
|
||||
"database",
|
||||
database,
|
||||
"billingBucketUrl",
|
||||
billingBucketUrl));
|
||||
LaunchFlexTemplateResponse launchResponse =
|
||||
|
||||
@@ -259,8 +259,8 @@ public class IcannReportingStager {
|
||||
tld,
|
||||
Ascii.toLowerCase(reportType.toString()),
|
||||
DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, reportFilename);
|
||||
final BlobId gcsFilename =
|
||||
BlobId.of(reportingBucket, String.format("%s/%s", subdir, reportFilename));
|
||||
gcsUtils.createFromBytes(gcsFilename, reportBytes);
|
||||
logger.atInfo().log("Wrote %d bytes to file location %s", reportBytes.length, gcsFilename);
|
||||
return reportFilename;
|
||||
@@ -268,8 +268,8 @@ public class IcannReportingStager {
|
||||
|
||||
/** Creates and stores a manifest file on GCS, indicating which reports were generated. */
|
||||
void createAndUploadManifest(String subdir, ImmutableList<String> filenames) throws IOException {
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, MANIFEST_FILE_NAME);
|
||||
final BlobId gcsFilename =
|
||||
BlobId.of(reportingBucket, String.format("%s/%s", subdir, MANIFEST_FILE_NAME));
|
||||
StringBuilder manifestString = new StringBuilder();
|
||||
filenames.forEach((filename) -> manifestString.append(filename).append("\n"));
|
||||
gcsUtils.createFromBytes(gcsFilename, manifestString.toString().getBytes(UTF_8));
|
||||
|
||||
@@ -145,10 +145,10 @@ public final class IcannReportingUploadAction implements Runnable {
|
||||
String.format(
|
||||
"icann/monthly/%d-%02d",
|
||||
cursorTimeMinusMonth.getYear(), cursorTimeMinusMonth.getMonthOfYear());
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, reportSubdir);
|
||||
String filename = getFileName(cursorType, cursorTime, tldStr);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, filename);
|
||||
logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname);
|
||||
final BlobId gcsFilename =
|
||||
BlobId.of(reportingBucket, String.format("%s/%s", reportSubdir, filename));
|
||||
logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportingBucket);
|
||||
// Check that the report exists
|
||||
try {
|
||||
verifyFileExists(gcsFilename);
|
||||
|
||||
@@ -28,17 +28,11 @@ import java.lang.reflect.Method;
|
||||
public class ReplaySpecializer {
|
||||
|
||||
public static void beforeSqlDelete(VKey<?> key) {
|
||||
invokeMethod(key.getKind(), "beforeSqlDelete", key);
|
||||
}
|
||||
|
||||
public static void beforeSqlSave(SqlEntity sqlEntity) {
|
||||
invokeMethod(sqlEntity.getClass(), "beforeSqlSave", sqlEntity);
|
||||
}
|
||||
|
||||
private static <T> void invokeMethod(Class<T> clazz, String methodName, Object argument) {
|
||||
String methodName = "beforeSqlDelete";
|
||||
Class<?> clazz = key.getKind();
|
||||
try {
|
||||
Method method = clazz.getMethod(methodName, argument.getClass());
|
||||
method.invoke(null, argument);
|
||||
Method method = clazz.getMethod(methodName, VKey.class);
|
||||
method.invoke(null, key);
|
||||
} catch (NoSuchMethodException e) {
|
||||
// Ignore, this just means that the class doesn't need this hook.
|
||||
} catch (IllegalAccessException e) {
|
||||
|
||||
@@ -26,4 +26,7 @@ import java.util.Optional;
|
||||
public interface SqlEntity {
|
||||
|
||||
Optional<DatastoreEntity> toDatastoreEntity();
|
||||
|
||||
/** A method that will ber called before the object is saved to SQL in asynchronous replay. */
|
||||
default void beforeSqlSaveOnReplay() {}
|
||||
}
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ import javax.inject.Inject;
|
||||
* However, since this is meant to be run during the Datastore-primary, SQL-secondary stage of the
|
||||
* migration, we want to make sure that we are using the most up-to-date version of the data. The
|
||||
* resource field of the history objects will be populated during asynchronous migration, e.g. in
|
||||
* {@link DomainHistory#beforeSqlSave(DomainHistory)}.
|
||||
* {@link DomainHistory#beforeSqlSaveOnReplay}.
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
|
||||
@@ -65,7 +65,6 @@
|
||||
<class>google.registry.model.registry.Registry</class>
|
||||
<class>google.registry.model.reporting.DomainTransactionRecord</class>
|
||||
<class>google.registry.model.reporting.Spec11ThreatMatch</class>
|
||||
<class>google.registry.model.server.KmsSecretRevision</class>
|
||||
<class>google.registry.model.server.Lock</class>
|
||||
<class>google.registry.model.server.ServerSecret</class>
|
||||
<class>google.registry.model.smd.SignedMarkRevocationList</class>
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
-- Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
--
|
||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
||||
-- you may not use this file except in compliance with the License.
|
||||
-- You may obtain a copy of the License at
|
||||
--
|
||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
||||
--
|
||||
-- Unless required by applicable law or agreed to in writing, software
|
||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
-- See the License for the specific language governing permissions and
|
||||
-- limitations under the License.
|
||||
|
||||
-- This query gathers all non-canceled billing events for a given
|
||||
-- YEAR_MONTH in yyyy-MM format.
|
||||
|
||||
-- This query differs from the one for BigQuery because JPQL does not have all
|
||||
-- the same functionality as a native query. Since there is no JPQL equivalent
|
||||
-- for REGEXP_EXTRACT the tld cannot be extracted from the targetId in the
|
||||
-- BillingEvent table. This requires an additional join with the domain table in
|
||||
-- order to find the value for the TLD associated with the billing event. Also,
|
||||
-- JPQL does not allow subqueries in the SELECT and FROM clauses. This prevents
|
||||
-- the query from being able to filter out certain data early in the query to
|
||||
-- reduce the size of intermediate results. It may be useful to measure this
|
||||
-- query's performance and consider switching to using a native query.
|
||||
|
||||
SELECT b, r FROM BillingEvent b
|
||||
JOIN Registrar r ON b.clientId = r.clientIdentifier
|
||||
JOIN Domain d ON b.domainRepoId = d.repoId
|
||||
JOIN Tld t ON t.tldStrId = d.tld
|
||||
LEFT JOIN BillingCancellation c ON b.id = c.refOneTime.billingId
|
||||
LEFT JOIN BillingCancellation cr ON b.cancellationMatchingBillingEvent = cr.refRecurring.billingId
|
||||
WHERE r.billingIdentifier IS NOT NULL
|
||||
AND r.type = 'REAL'
|
||||
AND t.invoicingEnabled IS TRUE
|
||||
AND b.billingTime BETWEEN CAST('%FIRST_TIMESTAMP_OF_MONTH%' AS timestamp) AND CAST('%LAST_TIMESTAMP_OF_MONTH%' AS timestamp)
|
||||
AND c.id IS NULL
|
||||
AND cr.id IS NULL
|
||||
@@ -61,6 +61,14 @@
|
||||
"regexes": [
|
||||
"^gs:\\/\\/[^\\n\\r]+$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "database",
|
||||
"label": "Database to read from.",
|
||||
"helpText": "DATASTORE or CLOUD_SQL.",
|
||||
"regexes": [
|
||||
"^DATASTORE|CLOUD_SQL$"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -14,10 +14,26 @@
|
||||
"name": "validationMode",
|
||||
"label": "How strict the marshaller validates the given EPP resources.",
|
||||
"helpText": "If set to LENIENT the marshaller will not warn about missing data on the EPP resources.",
|
||||
"is_optional": true,
|
||||
"regexes": [
|
||||
"^STRICT|LENIENT$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gcsBucket",
|
||||
"label": "The GCS bucket that where the resulting files will be stored.",
|
||||
"helpText": "Only the bucket name itself, without the leading \"gs://\".",
|
||||
"is_optional": false,
|
||||
"regexes": [
|
||||
"^[a-zA-Z0-9_\\-]+$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stagingKey",
|
||||
"label": "The PGP public key used to encrypt the RDE/BRDA deposit files.",
|
||||
"helpText": "The key is Base64 URL-safe encoded.",
|
||||
"regexes": [
|
||||
"A-Za-z0-9\\-_"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -25,10 +25,10 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
|
||||
@@ -18,6 +18,7 @@ import static com.google.common.collect.Iterables.transform;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static google.registry.backup.GcsDiffFileLister.getCommitLogDiffPrefix;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
@@ -28,13 +29,13 @@ import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.flogger.LoggerConfig;
|
||||
import com.google.common.testing.TestLogHandler;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.Executors;
|
||||
@@ -61,7 +62,7 @@ public class GcsDiffFileListerTest {
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
diffLister.gcsUtils = gcsUtils;
|
||||
diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
|
||||
diffLister.executorProvider = MoreExecutors::newDirectExecutorService;
|
||||
diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
addGcsFile(i, i + 1);
|
||||
@@ -189,12 +190,23 @@ public class GcsDiffFileListerTest {
|
||||
|
||||
@Test
|
||||
void testList_toTimeSpecified() {
|
||||
assertThat(listDiffFiles(
|
||||
now.minusMinutes(4).minusSeconds(1), now.minusMinutes(2).plusSeconds(1)))
|
||||
.containsExactly(
|
||||
now.minusMinutes(4),
|
||||
now.minusMinutes(3),
|
||||
now.minusMinutes(2))
|
||||
assertThat(
|
||||
listDiffFiles(now.minusMinutes(4).minusSeconds(1), now.minusMinutes(2).plusSeconds(1)))
|
||||
.containsExactly(now.minusMinutes(4), now.minusMinutes(3), now.minusMinutes(2))
|
||||
.inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testPrefix_lengthened() {
|
||||
DateTime from = DateTime.parse("2021-05-11T06:48:00.070Z");
|
||||
assertThat(getCommitLogDiffPrefix(from, null)).isEqualTo("commit_diff_until_");
|
||||
assertThat(getCommitLogDiffPrefix(from, DateTime.parse("2021-07-01")))
|
||||
.isEqualTo("commit_diff_until_2021-");
|
||||
assertThat(getCommitLogDiffPrefix(from, DateTime.parse("2021-05-21")))
|
||||
.isEqualTo("commit_diff_until_2021-05-");
|
||||
assertThat(getCommitLogDiffPrefix(from, DateTime.parse("2021-05-11T09:48:00.070Z")))
|
||||
.isEqualTo("commit_diff_until_2021-05-11T");
|
||||
assertThat(getCommitLogDiffPrefix(from, DateTime.parse("2021-05-11T06:59:00.070Z")))
|
||||
.isEqualTo("commit_diff_until_2021-05-11T06:");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
@@ -44,7 +45,6 @@ import com.google.common.truth.Truth8;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
@@ -67,6 +67,7 @@ import google.registry.persistence.transaction.TransactionManagerFactory;
|
||||
import google.registry.schema.replay.SqlReplayCheckpoint;
|
||||
import google.registry.schema.tld.PremiumEntry;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.DatabaseHelper;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.TestObject;
|
||||
@@ -132,7 +133,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
action.gcsBucket = "gcs bucket";
|
||||
action.diffLister = new GcsDiffFileLister();
|
||||
action.diffLister.gcsUtils = gcsUtils;
|
||||
action.diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
|
||||
action.diffLister.executorProvider = MoreExecutors::newDirectExecutorService;
|
||||
action.diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
|
||||
ofyTm()
|
||||
.transact(
|
||||
@@ -149,7 +150,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
ofyTm().transact(() -> DatabaseMigrationStateSchedule.set(DEFAULT_TRANSITION_MAP.toValueMap()));
|
||||
DatabaseHelper.removeDatabaseMigrationSchedule();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -200,7 +201,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
CommitLogMutation.create(manifest2Key, TestObject.create("f")));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
fakeClock.advanceOneMilli();
|
||||
runAndAssertSuccess(now);
|
||||
runAndAssertSuccess(now, 2);
|
||||
assertExpectedIds("previous to keep", "b", "d", "e", "f");
|
||||
}
|
||||
|
||||
@@ -211,7 +212,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
saveDiffFile(gcsUtils, createCheckpoint(now.minusMillis(2)));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMillis(1)));
|
||||
runAndAssertSuccess(now.minusMillis(1));
|
||||
runAndAssertSuccess(now.minusMillis(1), 0);
|
||||
assertExpectedIds("previous to keep");
|
||||
}
|
||||
|
||||
@@ -235,8 +236,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
assertThat(response.getStatus()).isEqualTo(SC_OK);
|
||||
assertThat(response.getPayload())
|
||||
.isEqualTo(
|
||||
"Running in dry-run mode; would have processed %d files. They are (limit 10):\n"
|
||||
+ "commit_diff_until_1999-12-31T23:59:00.000Z");
|
||||
"Running in dry-run mode, the first set of commit log files processed would be from "
|
||||
+ "searching from 1999-12-31T23:59:00.000Z to 1999-12-31T23:59:59.999Z and would "
|
||||
+ "contain 1 file(s). They are (limit 10): \n"
|
||||
+ "[commit_diff_until_1999-12-31T23:59:00.000Z]");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -253,7 +256,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("b")));
|
||||
runAndAssertSuccess(now.minusMinutes(1));
|
||||
runAndAssertSuccess(now.minusMinutes(1), 1);
|
||||
assertExpectedIds("previous to keep", "a", "b");
|
||||
}
|
||||
|
||||
@@ -275,7 +278,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
getBucketKey(1),
|
||||
now,
|
||||
ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
|
||||
runAndAssertSuccess(now.minusMinutes(1));
|
||||
runAndAssertSuccess(now.minusMinutes(1), 1);
|
||||
assertExpectedIds("previous to keep");
|
||||
}
|
||||
|
||||
@@ -347,7 +350,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
domainMutation,
|
||||
contactMutation);
|
||||
|
||||
runAndAssertSuccess(now.minusMinutes(1));
|
||||
runAndAssertSuccess(now.minusMinutes(1), 1);
|
||||
// Verify two things:
|
||||
// 1. that the contact insert occurred before the domain insert (necessary for FK ordering)
|
||||
// even though the domain came first in the file
|
||||
@@ -390,7 +393,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(1).plusMillis(1), ImmutableSet.of()),
|
||||
contactMutation);
|
||||
runAndAssertSuccess(now.minusMinutes(1).plusMillis(1));
|
||||
runAndAssertSuccess(now.minusMinutes(1).plusMillis(1), 1);
|
||||
// Verify that the delete occurred first (because it was in the first transaction) even though
|
||||
// deletes have higher weight
|
||||
ArgumentCaptor<Object> putCaptor = ArgumentCaptor.forClass(Object.class);
|
||||
@@ -435,7 +438,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
runAndAssertSuccess(now.minusMinutes(1));
|
||||
runAndAssertSuccess(now.minusMinutes(1), 1);
|
||||
// jpaTm()::put should only have been called with the checkpoint
|
||||
verify(spy, times(2)).put(any(SqlReplayCheckpoint.class));
|
||||
verify(spy, times(2)).put(any());
|
||||
@@ -460,7 +463,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
// one object only exists in Datastore, one is dually-written (so isn't replicated)
|
||||
ImmutableSet.of(getCrossTldKey(), claimsListKey)));
|
||||
|
||||
runAndAssertSuccess(now.minusMinutes(1));
|
||||
runAndAssertSuccess(now.minusMinutes(1), 1);
|
||||
verify(spy, times(0)).delete(any(VKey.class));
|
||||
}
|
||||
|
||||
@@ -503,7 +506,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")));
|
||||
runAndAssertSuccess(now.minusMinutes(1));
|
||||
runAndAssertSuccess(now.minusMinutes(1), 1);
|
||||
assertThat(TestObject.beforeSqlSaveCallCount).isEqualTo(1);
|
||||
}
|
||||
|
||||
@@ -522,11 +525,86 @@ public class ReplayCommitLogsToSqlActionTest {
|
||||
assertThat(TestObject.beforeSqlDeleteCallCount).isEqualTo(1);
|
||||
}
|
||||
|
||||
private void runAndAssertSuccess(DateTime expectedCheckpointTime) {
|
||||
@Test
|
||||
void testSuccess_cascadingDelete() throws Exception {
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
createTld("tld");
|
||||
DomainBase domain =
|
||||
newDomainBase("example.tld")
|
||||
.asBuilder()
|
||||
.setDsData(ImmutableSet.of(DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2})))
|
||||
.build();
|
||||
jpaTm().transact(() -> jpaTm().put(domain));
|
||||
|
||||
assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(DelegationSignerData.class))).isNotEmpty();
|
||||
|
||||
saveDiffFile(
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(3), ImmutableSet.of(Key.create(domain))));
|
||||
runAndAssertSuccess(now.minusMinutes(1), 1);
|
||||
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
assertThat(jpaTm().loadAllOf(DomainBase.class)).isEmpty();
|
||||
assertThat(jpaTm().loadAllOf(DelegationSignerData.class)).isEmpty();
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReplay_deleteAndResaveCascade_withOtherDeletion_noErrors() throws Exception {
|
||||
createTld("tld");
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
|
||||
// Save a domain with a particular dsData in SQL as the base
|
||||
ImmutableSet<DelegationSignerData> dsData =
|
||||
ImmutableSet.of(DelegationSignerData.create(1, 2, 3, new byte[] {4, 5, 6}));
|
||||
DomainBase domainWithDsData =
|
||||
newDomainBase("example.tld").asBuilder().setDsData(dsData).build();
|
||||
jpaTm().transact(() -> jpaTm().put(domainWithDsData));
|
||||
|
||||
// Replay a version of that domain without the dsData
|
||||
Key<CommitLogManifest> manifestKeyOne =
|
||||
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(3));
|
||||
DomainBase domainWithoutDsData =
|
||||
domainWithDsData.asBuilder().setDsData(ImmutableSet.of()).build();
|
||||
CommitLogMutation domainWithoutDsDataMutation =
|
||||
ofyTm().transact(() -> CommitLogMutation.create(manifestKeyOne, domainWithoutDsData));
|
||||
|
||||
// Create an object (any object) to delete via replay to trigger Hibernate flush events
|
||||
TestObject testObject = TestObject.create("foo", "bar");
|
||||
jpaTm().transact(() -> jpaTm().put(testObject));
|
||||
|
||||
// Replay the original domain, with the original dsData
|
||||
Key<CommitLogManifest> manifestKeyTwo =
|
||||
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(2));
|
||||
CommitLogMutation domainWithOriginalDsDataMutation =
|
||||
ofyTm().transact(() -> CommitLogMutation.create(manifestKeyTwo, domainWithDsData));
|
||||
|
||||
// If we try to perform all the events in one transaction (cascade-removal of the dsData,
|
||||
// cascade-adding the dsData back in, and deleting any other random object), Hibernate will
|
||||
// throw an exception
|
||||
saveDiffFile(
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(3), ImmutableSet.of(Key.create(testObject))),
|
||||
domainWithoutDsDataMutation,
|
||||
CommitLogManifest.create(getBucketKey(1), now.minusMinutes(2), ImmutableSet.of()),
|
||||
domainWithOriginalDsDataMutation);
|
||||
runAndAssertSuccess(now.minusMinutes(1), 1);
|
||||
}
|
||||
|
||||
private void runAndAssertSuccess(DateTime expectedCheckpointTime, int numFiles) {
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_OK);
|
||||
assertThat(response.getPayload())
|
||||
.isEqualTo("ReplayCommitLogsToSqlAction completed successfully.");
|
||||
.isEqualTo(
|
||||
String.format("Caught up to current time after replaying %d file(s).", numFiles));
|
||||
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(expectedCheckpointTime);
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import static org.joda.time.DateTimeZone.UTC;
|
||||
import com.google.appengine.api.datastore.DatastoreServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
@@ -36,7 +37,6 @@ import com.google.common.primitives.Longs;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
@@ -88,7 +88,7 @@ public class RestoreCommitLogsActionTest {
|
||||
action.gcsBucketOverride = Optional.empty();
|
||||
action.diffLister = new GcsDiffFileLister();
|
||||
action.diffLister.gcsUtils = gcsUtils;
|
||||
action.diffLister.lazyExecutor = MoreExecutors::newDirectExecutorService;
|
||||
action.diffLister.executorProvider = MoreExecutors::newDirectExecutorService;
|
||||
action.diffLister.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.FakeSleeper;
|
||||
@@ -73,8 +74,7 @@ public class WipeOutCloudSqlActionTest {
|
||||
|
||||
@Test
|
||||
void run_projectAllowed() throws Exception {
|
||||
WipeOutCloudSqlAction action =
|
||||
new WipeOutCloudSqlAction("domain-registry-qa", () -> conn, response, retrier);
|
||||
WipeOutCloudSqlAction action = new WipeOutCloudSqlAction(() -> conn, response, retrier);
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_OK);
|
||||
verify(stmt, times(1)).executeQuery(anyString());
|
||||
@@ -84,18 +84,21 @@ public class WipeOutCloudSqlActionTest {
|
||||
|
||||
@Test
|
||||
void run_projectNotAllowed() {
|
||||
WipeOutCloudSqlAction action =
|
||||
new WipeOutCloudSqlAction("domain-registry", () -> conn, response, retrier);
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_FORBIDDEN);
|
||||
verifyNoInteractions(stmt);
|
||||
try {
|
||||
RegistryEnvironment.SANDBOX.setup();
|
||||
WipeOutCloudSqlAction action = new WipeOutCloudSqlAction(() -> conn, response, retrier);
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_FORBIDDEN);
|
||||
verifyNoInteractions(stmt);
|
||||
} finally {
|
||||
RegistryEnvironment.UNITTEST.setup();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void run_nonRetrieableFailure() throws Exception {
|
||||
doThrow(new SQLException()).when(conn).getMetaData();
|
||||
WipeOutCloudSqlAction action =
|
||||
new WipeOutCloudSqlAction("domain-registry-qa", () -> conn, response, retrier);
|
||||
WipeOutCloudSqlAction action = new WipeOutCloudSqlAction(() -> conn, response, retrier);
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
|
||||
verifyNoInteractions(stmt);
|
||||
@@ -104,8 +107,7 @@ public class WipeOutCloudSqlActionTest {
|
||||
@Test
|
||||
void run_retrieableFailure() throws Exception {
|
||||
when(conn.getMetaData()).thenThrow(new RuntimeException()).thenReturn(metaData);
|
||||
WipeOutCloudSqlAction action =
|
||||
new WipeOutCloudSqlAction("domain-registry-qa", () -> conn, response, retrier);
|
||||
WipeOutCloudSqlAction action = new WipeOutCloudSqlAction(() -> conn, response, retrier);
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_OK);
|
||||
verify(stmt, times(1)).executeQuery(anyString());
|
||||
|
||||
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import google.registry.beam.BeamActionTestBase;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.testing.FakeClock;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -35,12 +36,22 @@ class WipeOutDatastoreActionTest extends BeamActionTestBase {
|
||||
|
||||
@Test
|
||||
void run_projectNotAllowed() {
|
||||
WipeoutDatastoreAction action =
|
||||
new WipeoutDatastoreAction(
|
||||
"domain-registry", "us-central1", "gs://some-bucket", clock, response, dataflow);
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_FORBIDDEN);
|
||||
verifyNoInteractions(dataflow);
|
||||
try {
|
||||
RegistryEnvironment.SANDBOX.setup();
|
||||
WipeoutDatastoreAction action =
|
||||
new WipeoutDatastoreAction(
|
||||
"domain-registry-sandbox",
|
||||
"us-central1",
|
||||
"gs://some-bucket",
|
||||
clock,
|
||||
response,
|
||||
dataflow);
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(SC_FORBIDDEN);
|
||||
verifyNoInteractions(dataflow);
|
||||
} finally {
|
||||
RegistryEnvironment.UNITTEST.setup();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -14,11 +14,39 @@
|
||||
|
||||
package google.registry.beam.invoicing;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.registry.Registry.TldState.GENERAL_AVAILABILITY;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.removeTmOverrideForTest;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.setTmForTest;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.newRegistry;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import google.registry.beam.TestPipelineExtension;
|
||||
import google.registry.model.billing.BillingEvent.Cancellation;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.billing.BillingEvent.OneTime;
|
||||
import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.billing.BillingEvent.Recurring;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.persistence.transaction.JpaTestRules;
|
||||
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationTestExtension;
|
||||
import google.registry.testing.DatastoreEntityExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.TestDataHelper;
|
||||
import google.registry.util.ResourceUtils;
|
||||
import java.io.File;
|
||||
@@ -26,12 +54,22 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.Arrays;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.testing.PAssert;
|
||||
import org.apache.beam.sdk.transforms.Create;
|
||||
import org.apache.beam.sdk.transforms.MapElements;
|
||||
import org.apache.beam.sdk.transforms.PTransform;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.apache.beam.sdk.values.TypeDescriptor;
|
||||
import org.joda.money.CurrencyUnit;
|
||||
import org.joda.money.Money;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
@@ -61,7 +99,7 @@ class InvoicingPipelineTest {
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
2,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
@@ -76,7 +114,7 @@ class InvoicingPipelineTest {
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
3,
|
||||
ZonedDateTime.of(2017, 10, 2, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 9, 29, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
@@ -88,10 +126,10 @@ class InvoicingPipelineTest {
|
||||
"REPO-ID",
|
||||
5,
|
||||
"JPY",
|
||||
70.75,
|
||||
70.0,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
4,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"bestdomains",
|
||||
@@ -106,7 +144,7 @@ class InvoicingPipelineTest {
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
5,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"anotherRegistrar",
|
||||
@@ -118,10 +156,10 @@ class InvoicingPipelineTest {
|
||||
"REPO-ID",
|
||||
1,
|
||||
"USD",
|
||||
0,
|
||||
0.0,
|
||||
"SUNRISE ANCHOR_TENANT"),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
6,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
@@ -133,10 +171,10 @@ class InvoicingPipelineTest {
|
||||
"REPO-ID",
|
||||
0,
|
||||
"USD",
|
||||
0,
|
||||
0.0,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
7,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
@@ -148,49 +186,57 @@ class InvoicingPipelineTest {
|
||||
"REPO-ID",
|
||||
0,
|
||||
"USD",
|
||||
20,
|
||||
20.0,
|
||||
""));
|
||||
|
||||
private static final ImmutableMap<String, ImmutableList<String>> EXPECTED_DETAILED_REPORT_MAP =
|
||||
ImmutableMap.of(
|
||||
"invoice_details_2017-10_theRegistrar_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
"2,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "test,RENEW,mydomain2.test,REPO-ID,3,USD,20.50,",
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "test,RENEW,mydomain.test,REPO-ID,3,USD,20.50,",
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
"7,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "test,SERVER_STATUS,update-prohibited.test,REPO-ID,0,USD,20.00,",
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
"6,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "test,SERVER_STATUS,locked.test,REPO-ID,0,USD,0.00,"),
|
||||
"invoice_details_2017-10_theRegistrar_hello.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-02 00:00:00 UTC,2017-09-29 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "hello,CREATE,mydomain3.hello,REPO-ID,5,JPY,70.75,"),
|
||||
"3,2017-10-02 00:00:00 UTC,2017-09-29 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "hello,CREATE,mydomain3.hello,REPO-ID,5,JPY,70.00,"),
|
||||
"invoice_details_2017-10_bestdomains_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,bestdomains,456,116688,"
|
||||
"4,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,bestdomains,456,116688,"
|
||||
+ "test,RENEW,mydomain4.test,REPO-ID,1,USD,20.50,"),
|
||||
"invoice_details_2017-10_anotherRegistrar_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,anotherRegistrar,789,,"
|
||||
"5,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,anotherRegistrar,789,,"
|
||||
+ "test,CREATE,mydomain5.test,REPO-ID,1,USD,0.00,SUNRISE ANCHOR_TENANT"));
|
||||
|
||||
private static final ImmutableList<String> EXPECTED_INVOICE_OUTPUT =
|
||||
ImmutableList.of(
|
||||
"2017-10-01,2020-09-30,234,41.00,USD,10125,1,PURCHASE,theRegistrar - test,2,"
|
||||
+ "RENEW | TLD: test | TERM: 3-year,20.50,USD,",
|
||||
"2017-10-01,2022-09-30,234,70.75,JPY,10125,1,PURCHASE,theRegistrar - hello,1,"
|
||||
+ "CREATE | TLD: hello | TERM: 5-year,70.75,JPY,",
|
||||
"2017-10-01,2022-09-30,234,70.00,JPY,10125,1,PURCHASE,theRegistrar - hello,1,"
|
||||
+ "CREATE | TLD: hello | TERM: 5-year,70.00,JPY,",
|
||||
"2017-10-01,,234,20.00,USD,10125,1,PURCHASE,theRegistrar - test,1,"
|
||||
+ "SERVER_STATUS | TLD: test | TERM: 0-year,20.00,USD,",
|
||||
"2017-10-01,2018-09-30,456,20.50,USD,10125,1,PURCHASE,bestdomains - test,1,"
|
||||
+ "RENEW | TLD: test | TERM: 1-year,20.50,USD,116688");
|
||||
|
||||
@RegisterExtension
|
||||
@Order(Order.DEFAULT - 1)
|
||||
final transient DatastoreEntityExtension datastore = new DatastoreEntityExtension();
|
||||
|
||||
@RegisterExtension
|
||||
final TestPipelineExtension pipeline =
|
||||
TestPipelineExtension.create().enableAbandonedNodeEnforcement(true);
|
||||
|
||||
@RegisterExtension
|
||||
final JpaIntegrationTestExtension database =
|
||||
new JpaTestRules.Builder().withClock(new FakeClock()).buildIntegrationTestRule();
|
||||
|
||||
@TempDir Path tmpDir;
|
||||
|
||||
private final InvoicingPipelineOptions options =
|
||||
@@ -218,6 +264,37 @@ class InvoicingPipelineTest {
|
||||
pipeline.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_fullSqlPipeline() throws Exception {
|
||||
setTmForTest(jpaTm());
|
||||
setupCloudSql();
|
||||
options.setDatabase("CLOUD_SQL");
|
||||
InvoicingPipeline invoicingPipeline = new InvoicingPipeline(options);
|
||||
invoicingPipeline.setupPipeline(pipeline);
|
||||
pipeline.run(options).waitUntilFinish();
|
||||
// Verify invoice CSV
|
||||
ImmutableList<String> overallInvoice = resultFileContents("REG-INV-2017-10.csv");
|
||||
assertThat(overallInvoice.get(0))
|
||||
.isEqualTo(
|
||||
"StartDate,EndDate,ProductAccountKey,Amount,AmountCurrency,BillingProductCode,"
|
||||
+ "SalesChannel,LineItemType,UsageGroupingKey,Quantity,Description,UnitPrice,"
|
||||
+ "UnitPriceCurrency,PONumber");
|
||||
assertThat(overallInvoice.subList(1, overallInvoice.size()))
|
||||
.containsExactlyElementsIn(EXPECTED_INVOICE_OUTPUT);
|
||||
removeTmOverrideForTest();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_readFromCloudSql() throws Exception {
|
||||
setTmForTest(jpaTm());
|
||||
setupCloudSql();
|
||||
PCollection<BillingEvent> billingEvents = InvoicingPipeline.readFromCloudSql(options, pipeline);
|
||||
billingEvents = billingEvents.apply(new changeDomainRepo());
|
||||
PAssert.that(billingEvents).containsInAnyOrder(INPUT_EVENTS);
|
||||
pipeline.run().waitUntilFinish();
|
||||
removeTmOverrideForTest();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_saveInvoiceCsv() throws Exception {
|
||||
InvoicingPipeline.saveInvoiceCsv(billingEvents, options);
|
||||
@@ -247,6 +324,30 @@ class InvoicingPipelineTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_makeCloudSqlQuery() throws Exception {
|
||||
// Pipeline must be run due to the TestPipelineExtension
|
||||
pipeline.run().waitUntilFinish();
|
||||
// Test that comments are removed from the .sql file correctly
|
||||
assertThat(InvoicingPipeline.makeCloudSqlQuery("2017-10"))
|
||||
.isEqualTo(
|
||||
"\n"
|
||||
+ "SELECT b, r FROM BillingEvent b\n"
|
||||
+ "JOIN Registrar r ON b.clientId = r.clientIdentifier\n"
|
||||
+ "JOIN Domain d ON b.domainRepoId = d.repoId\n"
|
||||
+ "JOIN Tld t ON t.tldStrId = d.tld\n"
|
||||
+ "LEFT JOIN BillingCancellation c ON b.id = c.refOneTime.billingId\n"
|
||||
+ "LEFT JOIN BillingCancellation cr ON b.cancellationMatchingBillingEvent ="
|
||||
+ " cr.refRecurring.billingId\n"
|
||||
+ "WHERE r.billingIdentifier IS NOT NULL\n"
|
||||
+ "AND r.type = 'REAL'\n"
|
||||
+ "AND t.invoicingEnabled IS TRUE\n"
|
||||
+ "AND b.billingTime BETWEEN CAST('2017-10-01' AS timestamp) AND CAST('2017-11-01'"
|
||||
+ " AS timestamp)\n"
|
||||
+ "AND c.id IS NULL\n"
|
||||
+ "AND cr.id IS NULL\n");
|
||||
}
|
||||
|
||||
/** Returns the text contents of a file under the beamBucket/results directory. */
|
||||
private ImmutableList<String> resultFileContents(String filename) throws Exception {
|
||||
File resultFile =
|
||||
@@ -256,4 +357,263 @@ class InvoicingPipelineTest {
|
||||
return ImmutableList.copyOf(
|
||||
ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n"));
|
||||
}
|
||||
|
||||
private void setupCloudSql() {
|
||||
// Populate billing events in Cloud SQL to match existing test data for Datastore
|
||||
persistNewRegistrar("NewRegistrar");
|
||||
persistNewRegistrar("TheRegistrar");
|
||||
Registrar registrar1 = persistNewRegistrar("theRegistrar");
|
||||
registrar1 = registrar1.asBuilder().setBillingIdentifier(234L).build();
|
||||
persistResource(registrar1);
|
||||
Registrar registrar2 = persistNewRegistrar("bestdomains");
|
||||
registrar2 =
|
||||
registrar2
|
||||
.asBuilder()
|
||||
.setBillingIdentifier(456L)
|
||||
.setPoNumber(Optional.of("116688"))
|
||||
.build();
|
||||
persistResource(registrar2);
|
||||
Registrar registrar3 = persistNewRegistrar("anotherRegistrar");
|
||||
registrar3 = registrar3.asBuilder().setBillingIdentifier(789L).build();
|
||||
persistResource(registrar3);
|
||||
|
||||
Registry test =
|
||||
newRegistry("test", "_TEST", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
|
||||
.asBuilder()
|
||||
.setInvoicingEnabled(true)
|
||||
.build();
|
||||
persistResource(test);
|
||||
Registry hello =
|
||||
newRegistry("hello", "_HELLO", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
|
||||
.asBuilder()
|
||||
.setInvoicingEnabled(true)
|
||||
.build();
|
||||
persistResource(hello);
|
||||
|
||||
DomainBase domain1 = persistActiveDomain("mydomain.test");
|
||||
DomainBase domain2 = persistActiveDomain("mydomain2.test");
|
||||
DomainBase domain3 = persistActiveDomain("mydomain3.hello");
|
||||
DomainBase domain4 = persistActiveDomain("mydomain4.test");
|
||||
DomainBase domain5 = persistActiveDomain("mydomain5.test");
|
||||
DomainBase domain6 = persistActiveDomain("locked.test");
|
||||
DomainBase domain7 = persistActiveDomain("update-prohibited.test");
|
||||
|
||||
persistOneTimeBillingEvent(
|
||||
1, domain1, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
|
||||
persistOneTimeBillingEvent(
|
||||
2, domain2, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
|
||||
persistOneTimeBillingEvent(
|
||||
3,
|
||||
domain3,
|
||||
registrar1,
|
||||
Reason.CREATE,
|
||||
5,
|
||||
Money.ofMajor(CurrencyUnit.JPY, 70),
|
||||
DateTime.parse("2017-09-29T00:00:00.0Z"),
|
||||
DateTime.parse("2017-10-02T00:00:00.0Z"));
|
||||
persistOneTimeBillingEvent(
|
||||
4, domain4, registrar2, Reason.RENEW, 1, Money.of(CurrencyUnit.USD, 20.5));
|
||||
persistOneTimeBillingEvent(
|
||||
5,
|
||||
domain5,
|
||||
registrar3,
|
||||
Reason.CREATE,
|
||||
1,
|
||||
Money.of(CurrencyUnit.USD, 0),
|
||||
DateTime.parse("2017-10-04T00:00:00.0Z"),
|
||||
DateTime.parse("2017-10-04T00:00:00.0Z"),
|
||||
Flag.SUNRISE,
|
||||
Flag.ANCHOR_TENANT);
|
||||
persistOneTimeBillingEvent(
|
||||
6, domain6, registrar1, Reason.SERVER_STATUS, 0, Money.of(CurrencyUnit.USD, 0));
|
||||
persistOneTimeBillingEvent(
|
||||
7, domain7, registrar1, Reason.SERVER_STATUS, 0, Money.of(CurrencyUnit.USD, 20));
|
||||
|
||||
// Add billing event for a non-billable registrar
|
||||
Registrar registrar4 = persistNewRegistrar("noBillRegistrar");
|
||||
registrar4 = registrar4.asBuilder().setBillingIdentifier(null).build();
|
||||
persistResource(registrar4);
|
||||
DomainBase domain8 = persistActiveDomain("non-billable.test");
|
||||
persistOneTimeBillingEvent(
|
||||
8, domain8, registrar4, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
|
||||
|
||||
// Add billing event for a non-real registrar
|
||||
Registrar registrar5 = persistNewRegistrar("notRealRegistrar");
|
||||
registrar5 =
|
||||
registrar5
|
||||
.asBuilder()
|
||||
.setIanaIdentifier(null)
|
||||
.setBillingIdentifier(456L)
|
||||
.setType(Registrar.Type.OTE)
|
||||
.build();
|
||||
persistResource(registrar5);
|
||||
DomainBase domain9 = persistActiveDomain("not-real.test");
|
||||
persistOneTimeBillingEvent(
|
||||
9, domain9, registrar5, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
|
||||
|
||||
// Add billing event for a non-invoicing TLD
|
||||
createTld("nobill");
|
||||
DomainBase domain10 = persistActiveDomain("test.nobill");
|
||||
persistOneTimeBillingEvent(
|
||||
10, domain10, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
|
||||
|
||||
// Add billing event before October 2017
|
||||
DomainBase domain11 = persistActiveDomain("july.test");
|
||||
persistOneTimeBillingEvent(
|
||||
11,
|
||||
domain11,
|
||||
registrar1,
|
||||
Reason.CREATE,
|
||||
5,
|
||||
Money.ofMajor(CurrencyUnit.JPY, 70),
|
||||
DateTime.parse("2017-06-29T00:00:00.0Z"),
|
||||
DateTime.parse("2017-07-02T00:00:00.0Z"));
|
||||
|
||||
// Add a billing event with a corresponding cancellation
|
||||
DomainBase domain12 = persistActiveDomain("cancel.test");
|
||||
OneTime oneTime =
|
||||
persistOneTimeBillingEvent(
|
||||
12, domain12, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
|
||||
DomainHistory domainHistory = persistDomainHistory(domain12, registrar1);
|
||||
|
||||
Cancellation cancellation =
|
||||
new Cancellation()
|
||||
.asBuilder()
|
||||
.setId(1)
|
||||
.setClientId(registrar1.getClientId())
|
||||
.setDomainHistoryRevisionId(domainHistory.getId())
|
||||
.setEventTime(DateTime.parse("2017-10-05T00:00:00.0Z"))
|
||||
.setBillingTime(DateTime.parse("2017-10-04T00:00:00.0Z"))
|
||||
.setOneTimeEventKey(oneTime.createVKey())
|
||||
.setTargetId(domain12.getDomainName())
|
||||
.setReason(Reason.RENEW)
|
||||
.setParent(domainHistory)
|
||||
.build();
|
||||
persistResource(cancellation);
|
||||
|
||||
// Add billing event with a corresponding recurring billing event and cancellation
|
||||
DomainBase domain13 = persistActiveDomain("cancel-recurring.test");
|
||||
DomainHistory domainHistoryRecurring = persistDomainHistory(domain13, registrar1);
|
||||
|
||||
Recurring recurring =
|
||||
new Recurring()
|
||||
.asBuilder()
|
||||
.setClientId(registrar1.getClientId())
|
||||
.setRecurrenceEndTime(END_OF_TIME)
|
||||
.setId(1)
|
||||
.setParent(domainHistoryRecurring)
|
||||
.setTargetId(domain13.getDomainName())
|
||||
.setEventTime(DateTime.parse("2017-10-04T00:00:00.0Z"))
|
||||
.setReason(Reason.RENEW)
|
||||
.build();
|
||||
persistResource(recurring);
|
||||
OneTime oneTimeRecurring =
|
||||
persistOneTimeBillingEvent(
|
||||
13, domain13, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
|
||||
oneTimeRecurring =
|
||||
oneTimeRecurring
|
||||
.asBuilder()
|
||||
.setCancellationMatchingBillingEvent(recurring.createVKey())
|
||||
.setFlags(ImmutableSet.of(Flag.SYNTHETIC))
|
||||
.setSyntheticCreationTime(DateTime.parse("2017-10-03T00:00:00.0Z"))
|
||||
.build();
|
||||
persistResource(oneTimeRecurring);
|
||||
|
||||
Cancellation cancellationRecurring =
|
||||
new Cancellation()
|
||||
.asBuilder()
|
||||
.setId(2)
|
||||
.setClientId(registrar1.getClientId())
|
||||
.setDomainHistoryRevisionId(domainHistoryRecurring.getId())
|
||||
.setEventTime(DateTime.parse("2017-10-05T00:00:00.0Z"))
|
||||
.setBillingTime(DateTime.parse("2017-10-04T00:00:00.0Z"))
|
||||
.setRecurringEventKey(recurring.createVKey())
|
||||
.setTargetId(domain13.getDomainName())
|
||||
.setReason(Reason.RENEW)
|
||||
.setParent(domainHistoryRecurring)
|
||||
.build();
|
||||
persistResource(cancellationRecurring);
|
||||
}
|
||||
|
||||
private DomainHistory persistDomainHistory(DomainBase domainBase, Registrar registrar) {
|
||||
DomainHistory domainHistory =
|
||||
new DomainHistory.Builder()
|
||||
.setType(HistoryEntry.Type.DOMAIN_RENEW)
|
||||
.setModificationTime(DateTime.parse("2017-10-04T00:00:00.0Z"))
|
||||
.setDomain(domainBase)
|
||||
.setClientId(registrar.getClientId())
|
||||
.build();
|
||||
return persistResource(domainHistory);
|
||||
}
|
||||
|
||||
private OneTime persistOneTimeBillingEvent(
|
||||
int id, DomainBase domainBase, Registrar registrar, Reason reason, int years, Money money) {
|
||||
return persistOneTimeBillingEvent(
|
||||
id,
|
||||
domainBase,
|
||||
registrar,
|
||||
reason,
|
||||
years,
|
||||
money,
|
||||
DateTime.parse("2017-10-04T00:00:00.0Z"),
|
||||
DateTime.parse("2017-10-04T00:00:00.0Z"));
|
||||
}
|
||||
|
||||
private OneTime persistOneTimeBillingEvent(
|
||||
int id,
|
||||
DomainBase domainBase,
|
||||
Registrar registrar,
|
||||
Reason reason,
|
||||
int years,
|
||||
Money money,
|
||||
DateTime eventTime,
|
||||
DateTime billingTime,
|
||||
Flag... flags) {
|
||||
google.registry.model.billing.BillingEvent.OneTime.Builder billingEventBuilder =
|
||||
new OneTime()
|
||||
.asBuilder()
|
||||
.setId(id)
|
||||
.setBillingTime(billingTime)
|
||||
.setEventTime(eventTime)
|
||||
.setClientId(registrar.getClientId())
|
||||
.setReason(reason)
|
||||
.setTargetId(domainBase.getDomainName())
|
||||
.setDomainRepoId("REPO-ID")
|
||||
.setCost(money)
|
||||
.setFlags(Arrays.stream(flags).collect(toImmutableSet()))
|
||||
.setParent(persistDomainHistory(domainBase, registrar));
|
||||
|
||||
if (years > 0) {
|
||||
billingEventBuilder.setPeriodYears(years);
|
||||
}
|
||||
|
||||
return persistResource(billingEventBuilder.build());
|
||||
}
|
||||
|
||||
private static class changeDomainRepo
|
||||
extends PTransform<PCollection<BillingEvent>, PCollection<BillingEvent>> {
|
||||
@Override
|
||||
public PCollection<BillingEvent> expand(PCollection<BillingEvent> input) {
|
||||
return input.apply(
|
||||
"Map to invoicing key",
|
||||
MapElements.into(TypeDescriptor.of(BillingEvent.class))
|
||||
.via(
|
||||
billingEvent ->
|
||||
BillingEvent.create(
|
||||
billingEvent.id(),
|
||||
billingEvent.billingTime(),
|
||||
billingEvent.eventTime(),
|
||||
billingEvent.registrarId(),
|
||||
billingEvent.billingId(),
|
||||
billingEvent.poNumber(),
|
||||
billingEvent.tld(),
|
||||
billingEvent.action(),
|
||||
billingEvent.domain(),
|
||||
"REPO-ID",
|
||||
billingEvent.years(),
|
||||
billingEvent.currency(),
|
||||
billingEvent.amount(),
|
||||
billingEvent.flags())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ import static google.registry.model.common.Cursor.CursorType.RDE_STAGING;
|
||||
import static google.registry.model.rde.RdeMode.FULL;
|
||||
import static google.registry.model.rde.RdeMode.THIN;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.setTmForTest;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.rde.RdeResourceType.CONTACT;
|
||||
import static google.registry.rde.RdeResourceType.DOMAIN;
|
||||
@@ -37,33 +36,54 @@ import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistEppResource;
|
||||
import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
|
||||
import static google.registry.util.ResourceUtils.readResourceUtf8;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.Duration.standardDays;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import google.registry.beam.TestPipelineExtension;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.rde.RdeMode;
|
||||
import google.registry.model.rde.RdeRevision;
|
||||
import google.registry.model.rde.RdeRevision.RdeRevisionId;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.Registrar.State;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.persistence.transaction.JpaTestRules;
|
||||
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationTestExtension;
|
||||
import google.registry.persistence.transaction.TransactionManager;
|
||||
import google.registry.persistence.transaction.TransactionManagerFactory;
|
||||
import google.registry.rde.DepositFragment;
|
||||
import google.registry.rde.Ghostryde;
|
||||
import google.registry.rde.PendingDeposit;
|
||||
import google.registry.rde.RdeResourceType;
|
||||
import google.registry.testing.DatastoreEntityExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeKeyringModule;
|
||||
import java.io.IOException;
|
||||
import java.util.function.Function;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.testing.PAssert;
|
||||
import org.apache.beam.sdk.transforms.GroupByKey;
|
||||
import org.apache.beam.sdk.transforms.Create;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.bouncycastle.openpgp.PGPPrivateKey;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
@@ -84,21 +104,44 @@ public class RdePipelineTest {
|
||||
|
||||
private static final String HOST_NAME_PATTERN = "<rdeHost:name>(.*)</rdeHost:name>";
|
||||
|
||||
private static final ImmutableSetMultimap<String, PendingDeposit> PENDINGS =
|
||||
ImmutableSetMultimap.of(
|
||||
"pal",
|
||||
PendingDeposit.create(
|
||||
"pal", DateTime.parse("2000-01-01TZ"), FULL, RDE_STAGING, standardDays(1)),
|
||||
"pal",
|
||||
PendingDeposit.create(
|
||||
"pal", DateTime.parse("2000-01-01TZ"), THIN, RDE_STAGING, standardDays(1)),
|
||||
"fun",
|
||||
PendingDeposit.create(
|
||||
"fun", DateTime.parse("2000-01-01TZ"), FULL, RDE_STAGING, standardDays(1)));
|
||||
|
||||
// This is the default creation time for test data.
|
||||
private final FakeClock clock = new FakeClock(DateTime.parse("1999-12-31TZ"));
|
||||
|
||||
// This is teh default as-of time the RDE/BRDA job.
|
||||
private final DateTime now = DateTime.parse("2000-01-01TZ");
|
||||
|
||||
private final ImmutableSetMultimap<String, PendingDeposit> pendings =
|
||||
ImmutableSetMultimap.of(
|
||||
"soy",
|
||||
PendingDeposit.create("soy", now, FULL, RDE_STAGING, standardDays(1)),
|
||||
"soy",
|
||||
PendingDeposit.create("soy", now, THIN, RDE_STAGING, standardDays(1)),
|
||||
"fun",
|
||||
PendingDeposit.create("fun", now, FULL, RDE_STAGING, standardDays(1)));
|
||||
|
||||
private final ImmutableList<DepositFragment> brdaFragments =
|
||||
ImmutableList.of(
|
||||
DepositFragment.create(RdeResourceType.DOMAIN, "<rdeDomain:domain/>\n", ""),
|
||||
DepositFragment.create(RdeResourceType.REGISTRAR, "<rdeRegistrar:registrar/>\n", ""));
|
||||
|
||||
private final ImmutableList<DepositFragment> rdeFragments =
|
||||
ImmutableList.of(
|
||||
DepositFragment.create(RdeResourceType.DOMAIN, "<rdeDomain:domain/>\n", ""),
|
||||
DepositFragment.create(RdeResourceType.REGISTRAR, "<rdeRegistrar:registrar/>\n", ""),
|
||||
DepositFragment.create(RdeResourceType.CONTACT, "<rdeContact:contact/>\n", ""),
|
||||
DepositFragment.create(RdeResourceType.HOST, "<rdeHost:host/>\n", ""));
|
||||
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private final PGPPublicKey encryptionKey =
|
||||
new FakeKeyringModule().get().getRdeStagingEncryptionKey();
|
||||
|
||||
private final PGPPrivateKey decryptionKey =
|
||||
new FakeKeyringModule().get().getRdeStagingDecryptionKey();
|
||||
|
||||
private final RdePipelineOptions options =
|
||||
PipelineOptionsFactory.create().as(RdePipelineOptions.class);
|
||||
|
||||
// The pipeline runs in a different thread, which needs to be masqueraded as a GAE thread as well.
|
||||
@RegisterExtension
|
||||
@Order(Order.DEFAULT - 1)
|
||||
@@ -110,19 +153,13 @@ public class RdePipelineTest {
|
||||
|
||||
@RegisterExtension
|
||||
final TestPipelineExtension pipeline =
|
||||
TestPipelineExtension.create().enableAbandonedNodeEnforcement(true);
|
||||
|
||||
private final RdePipelineOptions options =
|
||||
PipelineOptionsFactory.create().as(RdePipelineOptions.class);
|
||||
TestPipelineExtension.fromOptions(options).enableAbandonedNodeEnforcement(true);
|
||||
|
||||
private RdePipeline rdePipeline;
|
||||
|
||||
private TransactionManager originalTm;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
originalTm = tm();
|
||||
setTmForTest(jpaTm());
|
||||
TransactionManagerFactory.setTmForTest(jpaTm());
|
||||
loadInitialData();
|
||||
|
||||
// Two real registrars have been created by loadInitialData(), named "New Registrar" and "The
|
||||
@@ -142,16 +179,24 @@ public class RdePipelineTest {
|
||||
testRegistrar.asBuilder().setState(State.ACTIVE).build(),
|
||||
monitoringRegistrar.asBuilder().setState(State.ACTIVE).build())));
|
||||
|
||||
createTld("pal");
|
||||
createTld("soy");
|
||||
createTld("fun");
|
||||
createTld("cat");
|
||||
|
||||
tm().transact(
|
||||
() -> {
|
||||
tm().put(Cursor.create(CursorType.BRDA, now, Registry.get("soy")));
|
||||
tm().put(Cursor.create(CursorType.RDE_STAGING, now, Registry.get("soy")));
|
||||
RdeRevision.saveRevision("soy", now, THIN, 0);
|
||||
RdeRevision.saveRevision("soy", now, FULL, 0);
|
||||
});
|
||||
|
||||
// Also persists a "contact1234" contact in the process.
|
||||
persistActiveDomain("hello.pal");
|
||||
persistActiveDomain("hello.soy");
|
||||
persistActiveDomain("kitty.fun");
|
||||
// Should not appear anywhere.
|
||||
persistActiveDomain("lol.cat");
|
||||
persistDeletedDomain("deleted.pal", DateTime.parse("1999-12-30TZ"));
|
||||
persistDeletedDomain("deleted.soy", DateTime.parse("1999-12-30TZ"));
|
||||
|
||||
persistActiveContact("contact456");
|
||||
|
||||
@@ -160,30 +205,31 @@ public class RdePipelineTest {
|
||||
clock.advanceBy(Duration.standardDays(2));
|
||||
persistEppResource(host.asBuilder().setHostName("new.host.test").build());
|
||||
|
||||
options.setPendings(encodePendings(PENDINGS));
|
||||
options.setPendings(encodePendings(pendings));
|
||||
// The EPP resources created in tests do not have all the fields populated, using a STRICT
|
||||
// validation mode will result in a lot of warnings during marshalling.
|
||||
options.setValidationMode("LENIENT");
|
||||
rdePipeline = new RdePipeline(options);
|
||||
options.setStagingKey(
|
||||
BaseEncoding.base64Url().encode(PgpHelper.convertPublicKeyToBytes(encryptionKey)));
|
||||
options.setGcsBucket("gcs-bucket");
|
||||
options.setJobName("rde-job");
|
||||
rdePipeline = new RdePipeline(options, gcsUtils);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
setTmForTest(originalTm);
|
||||
TransactionManagerFactory.removeTmOverrideForTest();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_encodeAndDecodePendingsMap() throws Exception {
|
||||
String encodedString = encodePendings(PENDINGS);
|
||||
assertThat(decodePendings(encodedString)).isEqualTo(PENDINGS);
|
||||
String encodedString = encodePendings(pendings);
|
||||
assertThat(decodePendings(encodedString)).isEqualTo(pendings);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_createFragments() {
|
||||
PAssert.that(
|
||||
rdePipeline
|
||||
.createFragments(pipeline)
|
||||
.apply("Group by PendingDeposit", GroupByKey.create()))
|
||||
PAssert.that(rdePipeline.createFragments(pipeline))
|
||||
.satisfies(
|
||||
kvs -> {
|
||||
kvs.forEach(
|
||||
@@ -202,7 +248,7 @@ public class RdePipelineTest {
|
||||
.anyMatch(
|
||||
domain ->
|
||||
// Deleted domain should not be included
|
||||
domain.equals("deleted.pal")
|
||||
domain.equals("deleted.soy")
|
||||
// Only domains on the pending deposit's tld should
|
||||
// appear.
|
||||
|| !kv.getKey()
|
||||
@@ -242,6 +288,116 @@ public class RdePipelineTest {
|
||||
pipeline.run().waitUntilFinish();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_persistData() throws Exception {
|
||||
PendingDeposit brdaKey =
|
||||
PendingDeposit.create("soy", now, THIN, CursorType.BRDA, Duration.standardDays(1));
|
||||
PendingDeposit rdeKey =
|
||||
PendingDeposit.create("soy", now, FULL, CursorType.RDE_STAGING, Duration.standardDays(1));
|
||||
|
||||
verifyFiles(ImmutableMap.of(brdaKey, brdaFragments, rdeKey, rdeFragments), false);
|
||||
|
||||
assertThat(gcsUtils.listFolderObjects("gcs-bucket", "rde-job/"))
|
||||
.containsExactly(
|
||||
"soy_2000-01-01_thin_S1_R1.xml.length",
|
||||
"soy_2000-01-01_thin_S1_R1.xml.ghostryde",
|
||||
"soy_2000-01-01_full_S1_R1.xml.length",
|
||||
"soy_2000-01-01_full_S1_R1.xml.ghostryde",
|
||||
"soy_2000-01-01_full_S1_R1-report.xml.ghostryde");
|
||||
|
||||
assertThat(loadCursorTime(CursorType.BRDA))
|
||||
.isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
|
||||
assertThat(loadRevision(now, THIN)).isEqualTo(1);
|
||||
|
||||
assertThat(loadCursorTime(CursorType.RDE_STAGING))
|
||||
.isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
|
||||
assertThat(loadRevision(now, FULL)).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_persistData_manual() throws Exception {
|
||||
PendingDeposit brdaKey = PendingDeposit.createInManualOperation("soy", now, THIN, "test/", 0);
|
||||
PendingDeposit rdeKey = PendingDeposit.createInManualOperation("soy", now, FULL, "test/", 0);
|
||||
|
||||
verifyFiles(ImmutableMap.of(brdaKey, brdaFragments, rdeKey, rdeFragments), true);
|
||||
|
||||
assertThat(gcsUtils.listFolderObjects("gcs-bucket", "rde-job/"))
|
||||
.containsExactly(
|
||||
"manual/test/soy_2000-01-01_thin_S1_R0.xml.length",
|
||||
"manual/test/soy_2000-01-01_thin_S1_R0.xml.ghostryde",
|
||||
"manual/test/soy_2000-01-01_full_S1_R0.xml.length",
|
||||
"manual/test/soy_2000-01-01_full_S1_R0.xml.ghostryde",
|
||||
"manual/test/soy_2000-01-01_full_S1_R0-report.xml.ghostryde");
|
||||
|
||||
assertThat(loadCursorTime(CursorType.BRDA)).isEquivalentAccordingToCompareTo(now);
|
||||
assertThat(loadRevision(now, THIN)).isEqualTo(0);
|
||||
|
||||
assertThat(loadCursorTime(CursorType.RDE_STAGING)).isEquivalentAccordingToCompareTo(now);
|
||||
assertThat(loadRevision(now, FULL)).isEqualTo(0);
|
||||
}
|
||||
|
||||
private void verifyFiles(
|
||||
ImmutableMap<PendingDeposit, Iterable<DepositFragment>> input, boolean manual)
|
||||
throws Exception {
|
||||
PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> fragments =
|
||||
pipeline.apply("Create Input", Create.of(input));
|
||||
rdePipeline.persistData(fragments);
|
||||
pipeline.run().waitUntilFinish();
|
||||
|
||||
String prefix = manual ? "rde-job/manual/test/" : "rde-job/";
|
||||
String revision = manual ? "R0" : "R1";
|
||||
|
||||
// BRDA
|
||||
String brdaOutputFile =
|
||||
decryptGhostrydeGcsFile(prefix + "soy_2000-01-01_thin_S1_" + revision + ".xml.ghostryde");
|
||||
assertThat(brdaOutputFile)
|
||||
.isEqualTo(
|
||||
readResourceUtf8(this.getClass(), "reducer_brda.xml")
|
||||
.replace("%RESEND%", manual ? "" : " resend=\"1\""));
|
||||
compareLength(brdaOutputFile, prefix + "soy_2000-01-01_thin_S1_" + revision + ".xml.length");
|
||||
|
||||
// RDE
|
||||
String rdeOutputFile =
|
||||
decryptGhostrydeGcsFile(prefix + "soy_2000-01-01_full_S1_" + revision + ".xml.ghostryde");
|
||||
assertThat(rdeOutputFile)
|
||||
.isEqualTo(
|
||||
readResourceUtf8(RdePipelineTest.class, "reducer_rde.xml")
|
||||
.replace("%RESEND%", manual ? "" : " resend=\"1\""));
|
||||
compareLength(rdeOutputFile, prefix + "soy_2000-01-01_full_S1_" + revision + ".xml.length");
|
||||
assertThat(
|
||||
decryptGhostrydeGcsFile(
|
||||
prefix + "soy_2000-01-01_full_S1_" + revision + "-report.xml.ghostryde"))
|
||||
.isEqualTo(
|
||||
readResourceUtf8(RdePipelineTest.class, "reducer_rde_report.xml")
|
||||
.replace("%RESEND%", manual ? "0" : "1"));
|
||||
}
|
||||
|
||||
private String decryptGhostrydeGcsFile(String filename) throws IOException {
|
||||
return new String(
|
||||
Ghostryde.decode(gcsUtils.readBytesFrom(BlobId.of("gcs-bucket", filename)), decryptionKey),
|
||||
UTF_8);
|
||||
}
|
||||
|
||||
private void compareLength(String outputFile, String lengthFilename) throws IOException {
|
||||
assertThat(String.valueOf(outputFile.getBytes(UTF_8).length))
|
||||
.isEqualTo(
|
||||
new String(gcsUtils.readBytesFrom(BlobId.of("gcs-bucket", lengthFilename)), UTF_8));
|
||||
}
|
||||
|
||||
private static int loadRevision(DateTime now, RdeMode mode) {
|
||||
return tm().transact(
|
||||
() ->
|
||||
tm().loadByKey(
|
||||
VKey.createSql(
|
||||
RdeRevision.class,
|
||||
RdeRevisionId.create("soy", now.toLocalDate(), mode)))
|
||||
.getRevision());
|
||||
}
|
||||
|
||||
private static DateTime loadCursorTime(CursorType type) {
|
||||
return tm().transact(() -> tm().loadByKey(Cursor.createVKey(type, "soy")).getCursorTime());
|
||||
}
|
||||
|
||||
private static Function<DepositFragment, String> getXmlElement(String pattern) {
|
||||
return (fragment) -> {
|
||||
Matcher matcher = Pattern.compile(pattern).matcher(fragment.xml());
|
||||
|
||||
@@ -18,8 +18,8 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.removeTmOverrideForTest;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.setTmForTest;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.testing.AppEngineExtension.makeRegistrar1;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveContact;
|
||||
@@ -49,7 +49,6 @@ import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
|
||||
import google.registry.model.reporting.Spec11ThreatMatchDao;
|
||||
import google.registry.persistence.transaction.JpaTestRules;
|
||||
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationTestExtension;
|
||||
import google.registry.persistence.transaction.TransactionManager;
|
||||
import google.registry.testing.DatastoreEntityExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeSleeper;
|
||||
@@ -230,7 +229,6 @@ class Spec11PipelineTest {
|
||||
}
|
||||
|
||||
private void setupCloudSql() {
|
||||
TransactionManager originalTm = tm();
|
||||
setTmForTest(jpaTm());
|
||||
persistNewRegistrar("TheRegistrar");
|
||||
persistNewRegistrar("NewRegistrar");
|
||||
@@ -271,7 +269,7 @@ class Spec11PipelineTest {
|
||||
persistResource(createDomain("no-email.com", "2A4BA9BBC-COM", registrar2, contact2));
|
||||
persistResource(
|
||||
createDomain("anti-anti-anti-virus.dev", "555666888-DEV", registrar3, contact3));
|
||||
setTmForTest(originalTm);
|
||||
removeTmOverrideForTest();
|
||||
}
|
||||
|
||||
private void verifySaveToGcs() throws Exception {
|
||||
|
||||
@@ -29,11 +29,11 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.export.ExportDomainListsAction.ExportDomainListsReducer;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
|
||||
@@ -55,6 +55,7 @@ class EppLoginTlsTest extends EppTestCase {
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
clock);
|
||||
|
||||
@@ -64,6 +64,7 @@ class FlowRunnerTest {
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
clock);
|
||||
|
||||
@@ -53,6 +53,7 @@ final class TlsCredentialsTest {
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
clock);
|
||||
|
||||
@@ -50,6 +50,7 @@ class CertificateCheckerTest {
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
fakeClock);
|
||||
@@ -214,23 +215,45 @@ class CertificateCheckerTest {
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> certificateChecker.checkCertificate("bad certificate string"));
|
||||
assertThat(thrown).hasMessageThat().isEqualTo("Unable to read given certificate.");
|
||||
assertThat(thrown).hasMessageThat().contains("Unable to read given certificate");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_isNearingExpiration_yesItIs() throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-09-20T00:00:00Z"));
|
||||
void test_getCertificate_throwsException_invalidCertificateString() {
|
||||
IllegalArgumentException thrown =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> certificateChecker.getCertificate("invalidCertificateString"));
|
||||
assertThat(thrown).hasMessageThat().contains("Unable to read given certificate");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_getCertificate_returnsCertificateObject() throws Exception {
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-09-02T00:00:00Z"),
|
||||
DateTime.parse("2021-10-01T00:00:00Z"))
|
||||
.cert();
|
||||
assertThat(certificateChecker.isNearingExpiration(certificate)).isTrue();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
assertThat(certificateChecker.getCertificate(certificateStr).equals(certificate)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_isNearingExpiration_noItsNot() throws Exception {
|
||||
void test_serializeCertificate_returnsCertificateString() throws Exception {
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-09-02T00:00:00Z"),
|
||||
DateTime.parse("2021-10-01T00:00:00Z"))
|
||||
.cert();
|
||||
assertThat(
|
||||
certificateChecker.getCertificate(certificateChecker.serializeCertificate(certificate)))
|
||||
.isEqualTo(certificate);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_shouldReceiveExpiringNotification_returnsFalse_greaterThan30() throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-07-20T00:00:00Z"));
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
@@ -238,7 +261,124 @@ class CertificateCheckerTest {
|
||||
DateTime.parse("2020-09-02T00:00:00Z"),
|
||||
DateTime.parse("2021-10-01T00:00:00Z"))
|
||||
.cert();
|
||||
assertThat(certificateChecker.isNearingExpiration(certificate)).isFalse();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
assertThat(certificateChecker.shouldReceiveExpiringNotification(null, certificateStr))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_shouldReceiveExpiringNotification_returnsFalse_hasExpired() throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-10-20T00:00:00Z"));
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-09-02T00:00:00Z"),
|
||||
DateTime.parse("2021-10-01T00:00:00Z"))
|
||||
.cert();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
assertThat(certificateChecker.shouldReceiveExpiringNotification(null, certificateStr))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_shouldReceiveExpiringNotification_returnsTrue_on15days() throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-08-16T00:00:00Z"));
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-09-02T00:00:00Z"),
|
||||
DateTime.parse("2021-08-30T00:00:00Z"))
|
||||
.cert();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
DateTime lastExpiringNotificationSentDate = DateTime.parse("2021-08-01T00:00:00Z");
|
||||
assertThat(
|
||||
certificateChecker.shouldReceiveExpiringNotification(
|
||||
lastExpiringNotificationSentDate, certificateStr))
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_shouldReceiveExpiringNotification_returnsTrue_on30days() throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-01-01T00:00:00Z"));
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-01-02T00:00:00Z"),
|
||||
DateTime.parse("2021-01-30T00:00:00Z"))
|
||||
.cert();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
assertThat(certificateChecker.shouldReceiveExpiringNotification(null, certificateStr)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_shouldReceiveExpiringNotification_returnsFalse_between30and15_lastSentDateIsNotNull()
|
||||
throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-07-05T00:00:00Z"));
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-06-02T00:00:00Z"),
|
||||
DateTime.parse("2021-07-25T00:00:00Z"))
|
||||
.cert();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
DateTime lastExpiringNotificationSentDate = DateTime.parse("2021-07-04T00:00:00Z");
|
||||
assertThat(
|
||||
certificateChecker.shouldReceiveExpiringNotification(
|
||||
lastExpiringNotificationSentDate, certificateStr))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_shouldReceiveExpiringNotification_returnsTrue_between30and15_lastSentDateIsNull()
|
||||
throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-07-05T00:00:00Z"));
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-06-02T00:00:00Z"),
|
||||
DateTime.parse("2021-07-25T00:00:00Z"))
|
||||
.cert();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
assertThat(certificateChecker.shouldReceiveExpiringNotification(null, certificateStr)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void
|
||||
test_shouldReceiveExpiringNotification_returnsFalse_between15and0_lastSentDateBetween30and15_butLate()
|
||||
throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-07-05T00:00:00Z"));
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-06-02T00:00:00Z"),
|
||||
DateTime.parse("2021-07-18T00:00:00Z"))
|
||||
.cert();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
DateTime lastExpiringNotificationSentDate = DateTime.parse("2021-07-01T00:00:00Z");
|
||||
// The first notification date is late and difference between now and
|
||||
// lastExpiringNotificationSentDate is within expirationWarningIntervalDays.
|
||||
assertThat(
|
||||
certificateChecker.shouldReceiveExpiringNotification(
|
||||
lastExpiringNotificationSentDate, certificateStr))
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_shouldReceiveExpiringNotification_returnsTrue_between15and0_lastSentDateBetween30and15()
|
||||
throws Exception {
|
||||
fakeClock.setTo(DateTime.parse("2021-07-05T00:00:00Z"));
|
||||
X509Certificate certificate =
|
||||
SelfSignedCaCertificate.create(
|
||||
SSL_HOST,
|
||||
DateTime.parse("2020-06-02T00:00:00Z"),
|
||||
DateTime.parse("2021-07-18T00:00:00Z"))
|
||||
.cert();
|
||||
String certificateStr = certificateChecker.serializeCertificate(certificate);
|
||||
DateTime lastExpiringNotificationSentDate = DateTime.parse("2021-06-20T00:00:00Z");
|
||||
assertThat(
|
||||
certificateChecker.shouldReceiveExpiringNotification(
|
||||
lastExpiringNotificationSentDate, certificateStr))
|
||||
.isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -50,6 +50,7 @@ public class LoginFlowViaTlsTest extends LoginFlowTestCase {
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
clock);
|
||||
|
||||
@@ -19,11 +19,11 @@ import static com.google.common.truth.Truth.assertThat;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
@@ -1,517 +0,0 @@
|
||||
/*
|
||||
* Copyright 2016 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package google.registry.gcs.backport;
|
||||
|
||||
import com.google.api.client.util.DateTime;
|
||||
import com.google.api.services.storage.model.Bucket;
|
||||
import com.google.api.services.storage.model.ServiceAccount;
|
||||
import com.google.api.services.storage.model.StorageObject;
|
||||
import com.google.cloud.Tuple;
|
||||
import com.google.cloud.storage.Storage;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.cloud.storage.spi.v1.StorageRpc;
|
||||
import com.google.cloud.storage.testing.StorageRpcTestBase;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
/**
|
||||
* A bare-bones in-memory implementation of StorageRpc, meant for testing.
|
||||
*
|
||||
* <p>This class is <i>not</i> thread-safe. It's also (currently) limited in the following ways:
|
||||
*
|
||||
* <ul>
|
||||
* <li>Supported
|
||||
* <ul>
|
||||
* <li>object create
|
||||
* <li>object get
|
||||
* <li>object delete
|
||||
* <li>list the contents of a bucket
|
||||
* <li>generations
|
||||
* </ul>
|
||||
* <li>Unsupported
|
||||
* <ul>
|
||||
* <li>bucket create
|
||||
* <li>bucket get
|
||||
* <li>bucket delete
|
||||
* <li>list all buckets
|
||||
* <li>file attributes
|
||||
* <li>patch
|
||||
* <li>continueRewrite
|
||||
* <li>createBatch
|
||||
* <li>checksums, etags
|
||||
* <li>IAM operations
|
||||
* <li>BucketLock operations
|
||||
* <li>HMAC key operations
|
||||
* </ul>
|
||||
* </ul>
|
||||
*/
|
||||
@NotThreadSafe
|
||||
class FakeStorageRpc extends StorageRpcTestBase {
|
||||
|
||||
private static final SimpleDateFormat RFC_3339_FORMATTER =
|
||||
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
|
||||
|
||||
// fullname -> metadata
|
||||
Map<String, StorageObject> metadata = new HashMap<>();
|
||||
// fullname -> contents
|
||||
Map<String, byte[]> contents = new HashMap<>();
|
||||
// fullname -> future contents that will be visible on close.
|
||||
Map<String, byte[]> futureContents = new HashMap<>();
|
||||
|
||||
private final boolean throwIfOption;
|
||||
|
||||
/** @param throwIfOption if true, we throw when given any option */
|
||||
public FakeStorageRpc(boolean throwIfOption) {
|
||||
this.throwIfOption = throwIfOption;
|
||||
}
|
||||
|
||||
// remove all files
|
||||
void reset() {
|
||||
metadata = new HashMap<>();
|
||||
contents = new HashMap<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject create(StorageObject object, InputStream content, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
String key = fullname(object);
|
||||
object.setUpdated(now());
|
||||
metadata.put(key, object);
|
||||
try {
|
||||
contents.put(key, com.google.common.io.ByteStreams.toByteArray(content));
|
||||
} catch (IOException e) {
|
||||
throw new StorageException(e);
|
||||
}
|
||||
// TODO: crc, etc
|
||||
return object;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
String delimiter = null;
|
||||
String preprefix = "";
|
||||
String pageToken = null;
|
||||
long maxResults = Long.MAX_VALUE;
|
||||
for (Map.Entry<Option, ?> e : options.entrySet()) {
|
||||
switch (e.getKey()) {
|
||||
case PAGE_TOKEN:
|
||||
pageToken = (String) e.getValue();
|
||||
break;
|
||||
case PREFIX:
|
||||
preprefix = (String) e.getValue();
|
||||
if (preprefix.startsWith("/")) {
|
||||
preprefix = preprefix.substring(1);
|
||||
}
|
||||
break;
|
||||
case DELIMITER:
|
||||
delimiter = (String) e.getValue();
|
||||
break;
|
||||
case FIELDS:
|
||||
// ignore and return all the fields
|
||||
break;
|
||||
case MAX_RESULTS:
|
||||
maxResults = (Long) e.getValue();
|
||||
break;
|
||||
case USER_PROJECT:
|
||||
// prevent unsupported operation
|
||||
break;
|
||||
default:
|
||||
throw new UnsupportedOperationException("Unknown option: " + e.getKey());
|
||||
}
|
||||
}
|
||||
final String prefix = preprefix;
|
||||
|
||||
List<StorageObject> values = new ArrayList<>();
|
||||
Map<String, StorageObject> folders = new HashMap<>();
|
||||
for (StorageObject so : metadata.values()) {
|
||||
if (!so.getBucket().equals(bucket) || !so.getName().startsWith(prefix)) {
|
||||
continue;
|
||||
}
|
||||
if (processedAsFolder(so, delimiter, prefix, folders)) {
|
||||
continue;
|
||||
}
|
||||
so.setSize(size(so));
|
||||
values.add(so);
|
||||
}
|
||||
values.addAll(folders.values());
|
||||
|
||||
// truncate to max allowed length
|
||||
if (values.size() > maxResults) {
|
||||
List<StorageObject> newValues = new ArrayList<>();
|
||||
for (int i = 0; i < maxResults; i++) {
|
||||
newValues.add(values.get(i));
|
||||
}
|
||||
values = newValues;
|
||||
}
|
||||
|
||||
// null cursor to indicate there is no more data (empty string would cause us to be called
|
||||
// again).
|
||||
// The type cast seems to be necessary to help Java's typesystem remember that collections are
|
||||
// iterable.
|
||||
return Tuple.of(pageToken, (Iterable<StorageObject>) values);
|
||||
}
|
||||
|
||||
/** Returns the requested bucket or {@code null} if not found. */
|
||||
@Override
|
||||
public Bucket get(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Returns the requested storage object or {@code null} if not found. */
|
||||
@Override
|
||||
public StorageObject get(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
// we allow the "ID" option because we need to, but then we give a whole answer anyways
|
||||
// because the caller won't mind the extra fields.
|
||||
if (throwIfOption
|
||||
&& !options.isEmpty()
|
||||
&& options.size() > 1
|
||||
&& options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
String key = fullname(object);
|
||||
if (metadata.containsKey(key)) {
|
||||
StorageObject ret = metadata.get(key);
|
||||
ret.setSize(size(ret));
|
||||
ret.setId(key);
|
||||
|
||||
return ret;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bucket patch(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject patch(StorageObject storageObject, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(object);
|
||||
contents.remove(key);
|
||||
return null != metadata.remove(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject compose(
|
||||
Iterable<StorageObject> sources, StorageObject target, Map<Option, ?> targetOptions)
|
||||
throws StorageException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] load(StorageObject storageObject, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(storageObject);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
return contents.get(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Tuple<String, byte[]> read(
|
||||
StorageObject from, Map<Option, ?> options, long zposition, int zbytes)
|
||||
throws StorageException {
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option op : options.keySet()) {
|
||||
if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) {
|
||||
generationMatch = (Long) options.get(op);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unknown option: " + op);
|
||||
}
|
||||
}
|
||||
String key = fullname(from);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
long position = zposition;
|
||||
int bytes = zbytes;
|
||||
if (position < 0) {
|
||||
position = 0;
|
||||
}
|
||||
byte[] full = contents.get(key);
|
||||
if ((int) position + bytes > full.length) {
|
||||
bytes = full.length - (int) position;
|
||||
}
|
||||
if (bytes <= 0) {
|
||||
// special case: you're trying to read past the end
|
||||
return Tuple.of("etag-goes-here", new byte[0]);
|
||||
}
|
||||
byte[] ret = new byte[bytes];
|
||||
System.arraycopy(full, (int) position, ret, 0, bytes);
|
||||
return Tuple.of("etag-goes-here", ret);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long read(
|
||||
StorageObject from, Map<Option, ?> options, long position, OutputStream outputStream) {
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option op : options.keySet()) {
|
||||
if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) {
|
||||
generationMatch = (Long) options.get(op);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unknown option: " + op);
|
||||
}
|
||||
}
|
||||
String key = fullname(from);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
if (position < 0) {
|
||||
position = 0;
|
||||
}
|
||||
byte[] full = contents.get(key);
|
||||
int bytes = (int) (full.length - position);
|
||||
if (bytes <= 0) {
|
||||
// special case: you're trying to read past the end
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
outputStream.write(full, (int) position, bytes);
|
||||
} catch (IOException e) {
|
||||
throw new StorageException(500, "Failed to write to file", e);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String open(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(object);
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option option : options.keySet()) {
|
||||
// this is a bit of a hack, since we don't implement generations.
|
||||
if (option == Option.IF_GENERATION_MATCH) {
|
||||
generationMatch = (Long) options.get(option);
|
||||
}
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
metadata.put(key, object);
|
||||
|
||||
return fullname(object);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String open(String signedURL) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(
|
||||
String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, int length, boolean last)
|
||||
throws StorageException {
|
||||
writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject writeWithResponse(
|
||||
String uploadId,
|
||||
byte[] toWrite,
|
||||
int toWriteOffset,
|
||||
long destOffset,
|
||||
int length,
|
||||
boolean last) {
|
||||
// this may have a lot more allocations than ideal, but it'll work.
|
||||
byte[] bytes;
|
||||
if (futureContents.containsKey(uploadId)) {
|
||||
bytes = futureContents.get(uploadId);
|
||||
if (bytes.length < length + destOffset) {
|
||||
byte[] newBytes = new byte[(int) (length + destOffset)];
|
||||
System.arraycopy(bytes, 0, newBytes, (int) 0, bytes.length);
|
||||
bytes = newBytes;
|
||||
}
|
||||
} else {
|
||||
bytes = new byte[(int) (length + destOffset)];
|
||||
}
|
||||
System.arraycopy(toWrite, toWriteOffset, bytes, (int) destOffset, length);
|
||||
// we want to mimic the GCS behavior that file contents are only visible on close.
|
||||
StorageObject storageObject = null;
|
||||
if (last) {
|
||||
contents.put(uploadId, bytes);
|
||||
futureContents.remove(uploadId);
|
||||
if (metadata.containsKey(uploadId)) {
|
||||
storageObject = metadata.get(uploadId);
|
||||
storageObject.setUpdated(now());
|
||||
Long generation = storageObject.getGeneration();
|
||||
if (null == generation) {
|
||||
generation = Long.valueOf(0);
|
||||
}
|
||||
storageObject.setGeneration(++generation);
|
||||
metadata.put(uploadId, storageObject);
|
||||
}
|
||||
} else {
|
||||
futureContents.put(uploadId, bytes);
|
||||
}
|
||||
return storageObject;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException {
|
||||
String sourceKey = fullname(rewriteRequest.source);
|
||||
|
||||
// a little hackish, just good enough for the tests to work.
|
||||
if (!contents.containsKey(sourceKey)) {
|
||||
throw new StorageException(404, "File not found: " + sourceKey);
|
||||
}
|
||||
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option option : rewriteRequest.targetOptions.keySet()) {
|
||||
// this is a bit of a hack, since we don't implement generations.
|
||||
if (option == Option.IF_GENERATION_MATCH) {
|
||||
generationMatch = (Long) rewriteRequest.targetOptions.get(option);
|
||||
}
|
||||
}
|
||||
|
||||
String destKey = fullname(rewriteRequest.target);
|
||||
|
||||
// if this is a new file, set generation to 1, else increment the existing generation
|
||||
long generation = 1;
|
||||
if (metadata.containsKey(destKey)) {
|
||||
generation = metadata.get(destKey).getGeneration() + 1;
|
||||
}
|
||||
|
||||
checkGeneration(destKey, generationMatch);
|
||||
|
||||
byte[] data = contents.get(sourceKey);
|
||||
|
||||
rewriteRequest.target.setGeneration(generation);
|
||||
rewriteRequest.target.setSize(BigInteger.valueOf(data.length));
|
||||
rewriteRequest.target.setUpdated(metadata.get(sourceKey).getUpdated());
|
||||
|
||||
metadata.put(destKey, rewriteRequest.target);
|
||||
|
||||
contents.put(destKey, Arrays.copyOf(data, data.length));
|
||||
return new RewriteResponse(
|
||||
rewriteRequest,
|
||||
rewriteRequest.target,
|
||||
data.length,
|
||||
true,
|
||||
"rewriteToken goes here",
|
||||
data.length);
|
||||
}
|
||||
|
||||
private static DateTime now() {
|
||||
return DateTime.parseRfc3339(RFC_3339_FORMATTER.format(new Date()));
|
||||
}
|
||||
|
||||
private String fullname(StorageObject so) {
|
||||
return (so.getBucket() + "/" + so.getName());
|
||||
}
|
||||
|
||||
private BigInteger size(StorageObject so) {
|
||||
String key = fullname(so);
|
||||
|
||||
if (contents.containsKey(key)) {
|
||||
return BigInteger.valueOf(contents.get(key).length);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private void potentiallyThrow(Map<Option, ?> options) throws UnsupportedOperationException {
|
||||
if (throwIfOption && !options.isEmpty()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw if we're asking for generation 0 and the file exists, or if the requested generation
|
||||
* number doesn't match what is asked.
|
||||
*
|
||||
* @param key
|
||||
* @param generationMatch
|
||||
*/
|
||||
private void checkGeneration(String key, Long generationMatch) {
|
||||
if (null == generationMatch) {
|
||||
return;
|
||||
}
|
||||
if (generationMatch == 0 && metadata.containsKey(key)) {
|
||||
throw new StorageException(new FileAlreadyExistsException(key));
|
||||
}
|
||||
if (generationMatch != 0) {
|
||||
Long generation = metadata.get(key).getGeneration();
|
||||
if (!generationMatch.equals(generation)) {
|
||||
throw new StorageException(
|
||||
404, "Generation mismatch. Requested " + generationMatch + " but got " + generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if this is a folder. Adds it to folders if it isn't already there.
|
||||
private static boolean processedAsFolder(
|
||||
StorageObject so,
|
||||
String delimiter,
|
||||
String prefix, /* inout */
|
||||
Map<String, StorageObject> folders) {
|
||||
if (delimiter == null) {
|
||||
return false;
|
||||
}
|
||||
int nextSlash = so.getName().indexOf(delimiter, prefix.length());
|
||||
if (nextSlash < 0) {
|
||||
return false;
|
||||
}
|
||||
String folderName = so.getName().substring(0, nextSlash + 1);
|
||||
if (folders.containsKey(folderName)) {
|
||||
return true;
|
||||
}
|
||||
StorageObject fakeFolder = new StorageObject();
|
||||
fakeFolder.setName(folderName);
|
||||
fakeFolder.setBucket(so.getBucket());
|
||||
fakeFolder.setGeneration(so.getGeneration());
|
||||
fakeFolder.set("isDirectory", true);
|
||||
fakeFolder.setSize(BigInteger.ZERO);
|
||||
folders.put(folderName, fakeFolder);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServiceAccount getServiceAccount(String projectId) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
/*
|
||||
* Copyright 2016 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package google.registry.gcs.backport;
|
||||
|
||||
import com.google.cloud.ServiceRpc;
|
||||
import com.google.cloud.spi.ServiceRpcFactory;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
|
||||
/**
|
||||
* Utility to create an in-memory storage configuration for testing. Storage options can be obtained
|
||||
* via the {@link #getOptions()} method. Returned options will point to FakeStorageRpc.
|
||||
*
|
||||
* <p>Note, the created in-memory storage configuration supports limited set of operations and is
|
||||
* <b>not</b> thread-safe:
|
||||
*
|
||||
* <ul>
|
||||
* <li>Supported operations
|
||||
* <ul>
|
||||
* <li>object create
|
||||
* <li>object get
|
||||
* <li>object delete
|
||||
* <li>list the contents of a bucket
|
||||
* </ul>
|
||||
* <li>Unsupported operations
|
||||
* <ul>
|
||||
* <li>bucket create
|
||||
* <li>bucket get
|
||||
* <li>bucket delete
|
||||
* <li>list all buckets
|
||||
* <li>generations
|
||||
* <li>file attributes
|
||||
* <li>patch
|
||||
* <li>continueRewrite
|
||||
* <li>createBatch
|
||||
* <li>checksums, etags
|
||||
* <li>IAM operations
|
||||
* </ul>
|
||||
* </ul>
|
||||
*
|
||||
* {@link FakeStorageRpc#list(String, java.util.Map)} lists all the objects that have been created
|
||||
* rather than the objects in the provided bucket. Since this class does not support creating,
|
||||
* listing and deleting buckets, the parameter bucket here is not actually used and on serves as a
|
||||
* placeholder.
|
||||
*/
|
||||
public final class LocalStorageHelper {
|
||||
|
||||
// used for testing. Will throw if you pass it an option.
|
||||
private static final FakeStorageRpc instance = new FakeStorageRpc(true);
|
||||
|
||||
private LocalStorageHelper() {}
|
||||
|
||||
/**
|
||||
* Returns a {@link StorageOptions} that use the static FakeStorageRpc instance, and resets it
|
||||
* first so you start from a clean slate. That instance will throw if you pass it any option.
|
||||
*/
|
||||
public static StorageOptions getOptions() {
|
||||
instance.reset();
|
||||
return StorageOptions.newBuilder()
|
||||
.setProjectId("fake-project-for-testing")
|
||||
.setServiceRpcFactory(new FakeStorageRpcFactory())
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link StorageOptions} that creates a new FakeStorageRpc instance with the given
|
||||
* option.
|
||||
*/
|
||||
public static StorageOptions customOptions(final boolean throwIfOptions) {
|
||||
return StorageOptions.newBuilder()
|
||||
.setProjectId("fake-project-for-testing")
|
||||
.setServiceRpcFactory(new FakeStorageRpcFactory(new FakeStorageRpc(throwIfOptions)))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static class FakeStorageRpcFactory implements ServiceRpcFactory<StorageOptions> {
|
||||
|
||||
private final FakeStorageRpc instance;
|
||||
|
||||
public FakeStorageRpcFactory() {
|
||||
this(LocalStorageHelper.instance);
|
||||
}
|
||||
|
||||
public FakeStorageRpcFactory(FakeStorageRpc instance) {
|
||||
this.instance = instance;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServiceRpc create(StorageOptions storageOptions) {
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
The files here are directly lifted
|
||||
from [googleapis/java-storage-nio](https://github.com/googleapis/java-storage-nio/tree/master/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing)
|
||||
. They are needed because the `StorageOptions` returned from `LocalStorageHelper.getOptions()`
|
||||
should be serializable for the `GcsUtils` class itself to be serializable in tests, but is not. The
|
||||
bug is [fixed](https://github.com/googleapis/java-storage-nio/pull/606) upstream. However, the
|
||||
current released package does not contain the fix yet.
|
||||
|
||||
They are not put under `common/testing` because we do not want to introduce a dependency on the
|
||||
testing configuration from core.
|
||||
+2
-2
@@ -15,7 +15,6 @@
|
||||
package google.registry.model.common;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_ONLY;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY;
|
||||
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_READ_ONLY;
|
||||
@@ -30,6 +29,7 @@ import static org.junit.Assert.assertThrows;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import google.registry.model.EntityTestCase;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.testing.DatabaseHelper;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
@@ -45,7 +45,7 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
ofyTm().transact(() -> DatabaseMigrationStateSchedule.set(DEFAULT_TRANSITION_MAP.toValueMap()));
|
||||
DatabaseHelper.removeDatabaseMigrationSchedule();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.testing.DatabaseHelper.newContactResource;
|
||||
import static google.registry.testing.DatabaseHelper.newContactResourceWithRoid;
|
||||
import static google.registry.testing.SqlHelper.saveRegistrar;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
@@ -47,7 +48,7 @@ public class ContactHistoryTest extends EntityTestCase {
|
||||
jpaTm().transact(() -> jpaTm().insert(contact));
|
||||
VKey<ContactResource> contactVKey = contact.createVKey();
|
||||
ContactResource contactFromDb = jpaTm().transact(() -> jpaTm().loadByKey(contactVKey));
|
||||
ContactHistory contactHistory = createContactHistory(contactFromDb, contact.getRepoId());
|
||||
ContactHistory contactHistory = createContactHistory(contactFromDb);
|
||||
jpaTm().transact(() -> jpaTm().insert(contactHistory));
|
||||
jpaTm()
|
||||
.transact(
|
||||
@@ -67,10 +68,7 @@ public class ContactHistoryTest extends EntityTestCase {
|
||||
VKey<ContactResource> contactVKey = contact.createVKey();
|
||||
ContactResource contactFromDb = jpaTm().transact(() -> jpaTm().loadByKey(contactVKey));
|
||||
ContactHistory contactHistory =
|
||||
createContactHistory(contactFromDb, contact.getRepoId())
|
||||
.asBuilder()
|
||||
.setContact(null)
|
||||
.build();
|
||||
createContactHistory(contactFromDb).asBuilder().setContact(null).build();
|
||||
jpaTm().transact(() -> jpaTm().insert(contactHistory));
|
||||
|
||||
jpaTm()
|
||||
@@ -91,7 +89,7 @@ public class ContactHistoryTest extends EntityTestCase {
|
||||
VKey<ContactResource> contactVKey = contact.createVKey();
|
||||
ContactResource contactFromDb = tm().transact(() -> tm().loadByKey(contactVKey));
|
||||
fakeClock.advanceOneMilli();
|
||||
ContactHistory contactHistory = createContactHistory(contactFromDb, contact.getRepoId());
|
||||
ContactHistory contactHistory = createContactHistory(contactFromDb);
|
||||
tm().transact(() -> tm().insert(contactHistory));
|
||||
|
||||
// retrieving a HistoryEntry or a ContactHistory with the same key should return the same object
|
||||
@@ -106,7 +104,38 @@ public class ContactHistoryTest extends EntityTestCase {
|
||||
assertThat(hostHistoryFromDb).isEqualTo(historyEntryFromDb);
|
||||
}
|
||||
|
||||
private ContactHistory createContactHistory(ContactBase contact, String contactRepoId) {
|
||||
@Test
|
||||
void testBeforeSqlSave_afterContactPersisted() {
|
||||
saveRegistrar("TheRegistrar");
|
||||
ContactResource contactResource = newContactResource("contactId");
|
||||
ContactHistory contactHistory =
|
||||
new ContactHistory.Builder()
|
||||
.setType(HistoryEntry.Type.HOST_CREATE)
|
||||
.setXmlBytes("<xml></xml>".getBytes(UTF_8))
|
||||
.setModificationTime(fakeClock.nowUtc())
|
||||
.setClientId("TheRegistrar")
|
||||
.setTrid(Trid.create("ABC-123", "server-trid"))
|
||||
.setBySuperuser(false)
|
||||
.setReason("reason")
|
||||
.setRequestedByRegistrar(true)
|
||||
.setContactRepoId(contactResource.getRepoId())
|
||||
.build();
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
jpaTm().put(contactResource);
|
||||
contactHistory.beforeSqlSaveOnReplay();
|
||||
jpaTm().put(contactHistory);
|
||||
});
|
||||
jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
assertAboutImmutableObjects()
|
||||
.that(jpaTm().loadByEntity(contactResource))
|
||||
.hasFieldsEqualTo(jpaTm().loadByEntity(contactHistory).getContactBase().get()));
|
||||
}
|
||||
|
||||
private ContactHistory createContactHistory(ContactBase contact) {
|
||||
return new ContactHistory.Builder()
|
||||
.setType(HistoryEntry.Type.HOST_CREATE)
|
||||
.setXmlBytes("<xml></xml>".getBytes(UTF_8))
|
||||
@@ -117,7 +146,7 @@ public class ContactHistoryTest extends EntityTestCase {
|
||||
.setReason("reason")
|
||||
.setRequestedByRegistrar(true)
|
||||
.setContact(contact)
|
||||
.setContactRepoId(contactRepoId)
|
||||
.setContactRepoId(contact.getRepoId())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -190,6 +190,44 @@ public class DomainHistoryTest extends EntityTestCase {
|
||||
jpaTm().transact(() -> jpaTm().put(domainHistoryFromDb2));
|
||||
}
|
||||
|
||||
@TestSqlOnly
|
||||
void testBeforeSqlSave_afterDomainPersisted() {
|
||||
DomainBase domain = createDomainWithContactsAndHosts();
|
||||
DomainHistory historyWithoutResource =
|
||||
new DomainHistory.Builder()
|
||||
.setType(HistoryEntry.Type.DOMAIN_CREATE)
|
||||
.setXmlBytes("<xml></xml>".getBytes(UTF_8))
|
||||
.setModificationTime(fakeClock.nowUtc())
|
||||
.setClientId("TheRegistrar")
|
||||
.setTrid(Trid.create("ABC-123", "server-trid"))
|
||||
.setBySuperuser(false)
|
||||
.setReason("reason")
|
||||
.setRequestedByRegistrar(true)
|
||||
.setDomainRepoId(domain.getRepoId())
|
||||
.setOtherClientId("otherClient")
|
||||
.setPeriod(Period.create(1, Period.Unit.YEARS))
|
||||
.build();
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
jpaTm()
|
||||
.put(
|
||||
domain
|
||||
.asBuilder()
|
||||
.setPersistedCurrentSponsorClientId("NewRegistrar")
|
||||
.build());
|
||||
historyWithoutResource.beforeSqlSaveOnReplay();
|
||||
jpaTm().put(historyWithoutResource);
|
||||
});
|
||||
jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
assertAboutImmutableObjects()
|
||||
.that(jpaTm().loadByEntity(domain))
|
||||
.hasFieldsEqualTo(
|
||||
jpaTm().loadByEntity(historyWithoutResource).getDomainContent().get()));
|
||||
}
|
||||
|
||||
static DomainBase createDomainWithContactsAndHosts() {
|
||||
createTld("tld");
|
||||
HostResource host = newHostResourceWithRoid("ns1.example.com", "host1");
|
||||
|
||||
@@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.testing.DatabaseHelper.newHostResource;
|
||||
import static google.registry.testing.DatabaseHelper.newHostResourceWithRoid;
|
||||
import static google.registry.testing.SqlHelper.saveRegistrar;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
@@ -48,7 +49,7 @@ public class HostHistoryTest extends EntityTestCase {
|
||||
VKey<HostResource> hostVKey =
|
||||
VKey.create(HostResource.class, "host1", Key.create(HostResource.class, "host1"));
|
||||
HostResource hostFromDb = jpaTm().transact(() -> jpaTm().loadByKey(hostVKey));
|
||||
HostHistory hostHistory = createHostHistory(hostFromDb, host.getRepoId());
|
||||
HostHistory hostHistory = createHostHistory(hostFromDb);
|
||||
jpaTm().transact(() -> jpaTm().insert(hostHistory));
|
||||
jpaTm()
|
||||
.transact(
|
||||
@@ -66,8 +67,7 @@ public class HostHistoryTest extends EntityTestCase {
|
||||
jpaTm().transact(() -> jpaTm().insert(host));
|
||||
|
||||
HostResource hostFromDb = jpaTm().transact(() -> jpaTm().loadByKey(host.createVKey()));
|
||||
HostHistory hostHistory =
|
||||
createHostHistory(hostFromDb, host.getRepoId()).asBuilder().setHost(null).build();
|
||||
HostHistory hostHistory = createHostHistory(hostFromDb).asBuilder().setHost(null).build();
|
||||
jpaTm().transact(() -> jpaTm().insert(hostHistory));
|
||||
|
||||
jpaTm()
|
||||
@@ -88,7 +88,7 @@ public class HostHistoryTest extends EntityTestCase {
|
||||
VKey<HostResource> hostVKey =
|
||||
VKey.create(HostResource.class, "host1", Key.create(HostResource.class, "host1"));
|
||||
HostResource hostFromDb = tm().transact(() -> tm().loadByKey(hostVKey));
|
||||
HostHistory hostHistory = createHostHistory(hostFromDb, host.getRepoId());
|
||||
HostHistory hostHistory = createHostHistory(hostFromDb);
|
||||
fakeClock.advanceOneMilli();
|
||||
tm().transact(() -> tm().insert(hostHistory));
|
||||
|
||||
@@ -104,6 +104,37 @@ public class HostHistoryTest extends EntityTestCase {
|
||||
assertThat(hostHistoryFromDb).isEqualTo(historyEntryFromDb);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBeforeSqlSave_afterHostPersisted() {
|
||||
saveRegistrar("TheRegistrar");
|
||||
HostResource hostResource = newHostResource("ns1.example.tld");
|
||||
HostHistory hostHistory =
|
||||
new HostHistory.Builder()
|
||||
.setType(HistoryEntry.Type.HOST_CREATE)
|
||||
.setXmlBytes("<xml></xml>".getBytes(UTF_8))
|
||||
.setModificationTime(fakeClock.nowUtc())
|
||||
.setClientId("TheRegistrar")
|
||||
.setTrid(Trid.create("ABC-123", "server-trid"))
|
||||
.setBySuperuser(false)
|
||||
.setReason("reason")
|
||||
.setRequestedByRegistrar(true)
|
||||
.setHostRepoId(hostResource.getRepoId())
|
||||
.build();
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
jpaTm().put(hostResource);
|
||||
hostHistory.beforeSqlSaveOnReplay();
|
||||
jpaTm().put(hostHistory);
|
||||
});
|
||||
jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
assertAboutImmutableObjects()
|
||||
.that(jpaTm().loadByEntity(hostResource))
|
||||
.hasFieldsEqualTo(jpaTm().loadByEntity(hostHistory).getHostBase().get()));
|
||||
}
|
||||
|
||||
private void assertHostHistoriesEqual(HostHistory one, HostHistory two) {
|
||||
assertAboutImmutableObjects().that(one).isEqualExceptFields(two, "hostBase");
|
||||
assertAboutImmutableObjects()
|
||||
@@ -111,7 +142,7 @@ public class HostHistoryTest extends EntityTestCase {
|
||||
.isEqualExceptFields(two.getHostBase().orElse(null), "repoId");
|
||||
}
|
||||
|
||||
private HostHistory createHostHistory(HostBase hostBase, String hostRepoId) {
|
||||
private HostHistory createHostHistory(HostBase hostBase) {
|
||||
return new HostHistory.Builder()
|
||||
.setType(HistoryEntry.Type.HOST_CREATE)
|
||||
.setXmlBytes("<xml></xml>".getBytes(UTF_8))
|
||||
@@ -122,7 +153,7 @@ public class HostHistoryTest extends EntityTestCase {
|
||||
.setReason("reason")
|
||||
.setRequestedByRegistrar(true)
|
||||
.setHost(hostBase)
|
||||
.setHostRepoId(hostRepoId)
|
||||
.setHostRepoId(hostBase.getRepoId())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,7 +74,7 @@ public class LegacyHistoryObjectTest extends EntityTestCase {
|
||||
assertAboutImmutableObjects()
|
||||
.that(legacyHistoryEntry)
|
||||
.isEqualExceptFields(fromObjectify, "contactBase", "contactRepoId");
|
||||
assertThat(fromObjectify instanceof ContactHistory).isTrue();
|
||||
assertThat(fromObjectify).isInstanceOf(ContactHistory.class);
|
||||
ContactHistory legacyContactHistory = (ContactHistory) fromObjectify;
|
||||
|
||||
// Next, save that from-Datastore object in SQL and verify we can load it back in
|
||||
@@ -124,7 +124,7 @@ public class LegacyHistoryObjectTest extends EntityTestCase {
|
||||
"nsHosts",
|
||||
"dsDataHistories",
|
||||
"gracePeriodHistories");
|
||||
assertThat(fromObjectify instanceof DomainHistory).isTrue();
|
||||
assertThat(fromObjectify).isInstanceOf(DomainHistory.class);
|
||||
DomainHistory legacyDomainHistory = (DomainHistory) fromObjectify;
|
||||
|
||||
// Next, save that from-Datastore object in SQL and verify we can load it back in
|
||||
@@ -137,15 +137,7 @@ public class LegacyHistoryObjectTest extends EntityTestCase {
|
||||
// Don't compare nsHosts directly because one is null and the other is empty
|
||||
assertAboutImmutableObjects()
|
||||
.that(legacyDomainHistory)
|
||||
.isEqualExceptFields(
|
||||
// NB: period, transaction records, and other client ID are added in #794
|
||||
legacyHistoryFromSql,
|
||||
"period",
|
||||
"domainTransactionRecords",
|
||||
"otherClientId",
|
||||
"nsHosts",
|
||||
"dsDataHistories",
|
||||
"gracePeriodHistories");
|
||||
.isEqualExceptFields(legacyHistoryFromSql, "nsHosts");
|
||||
assertThat(nullToEmpty(legacyDomainHistory.getNsHosts()))
|
||||
.isEqualTo(nullToEmpty(legacyHistoryFromSql.getNsHosts()));
|
||||
});
|
||||
@@ -174,7 +166,7 @@ public class LegacyHistoryObjectTest extends EntityTestCase {
|
||||
assertAboutImmutableObjects()
|
||||
.that(legacyHistoryEntry)
|
||||
.isEqualExceptFields(fromObjectify, "hostBase", "hostRepoId");
|
||||
assertThat(fromObjectify instanceof HostHistory).isTrue();
|
||||
assertThat(fromObjectify).isInstanceOf(HostHistory.class);
|
||||
HostHistory legacyHostHistory = (HostHistory) fromObjectify;
|
||||
|
||||
// Next, save that from-Datastore object in SQL and verify we can load it back in
|
||||
|
||||
@@ -29,6 +29,7 @@ import static google.registry.testing.DatabaseHelper.newRegistry;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.DatabaseHelper.persistSimpleResource;
|
||||
import static google.registry.testing.DatabaseHelper.persistSimpleResources;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
@@ -414,6 +415,64 @@ class RegistrarTest extends EntityTestCase {
|
||||
IllegalArgumentException.class, () -> new Registrar.Builder().setPhonePasscode("code1"));
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_getLastExpiringCertNotificationSentDate_returnsInitialValue() {
|
||||
assertThat(registrar.getLastExpiringCertNotificationSentDate()).isEqualTo(START_OF_TIME);
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_getLastExpiringFailoverCertNotificationSentDate_returnsInitialValue() {
|
||||
assertThat(registrar.getLastExpiringFailoverCertNotificationSentDate())
|
||||
.isEqualTo(START_OF_TIME);
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_setLastExpiringCertNotificationSentDate() {
|
||||
assertThat(
|
||||
registrar
|
||||
.asBuilder()
|
||||
.setLastExpiringCertNotificationSentDate(fakeClock.nowUtc())
|
||||
.build()
|
||||
.getLastExpiringCertNotificationSentDate())
|
||||
.isEqualTo(fakeClock.nowUtc());
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testFailure_setLastExpiringCertNotificationSentDate_nullDate() {
|
||||
IllegalArgumentException thrown =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> new Registrar.Builder().setLastExpiringCertNotificationSentDate(null).build());
|
||||
assertThat(thrown)
|
||||
.hasMessageThat()
|
||||
.isEqualTo("Registrar lastExpiringCertNotificationSentDate cannot be null");
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_setLastExpiringFailoverCertNotificationSentDate() {
|
||||
assertThat(
|
||||
registrar
|
||||
.asBuilder()
|
||||
.setLastExpiringFailoverCertNotificationSentDate(fakeClock.nowUtc())
|
||||
.build()
|
||||
.getLastExpiringFailoverCertNotificationSentDate())
|
||||
.isEqualTo(fakeClock.nowUtc());
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testFailure_setLastExpiringFailoverCertNotificationSentDate_nullDate() {
|
||||
IllegalArgumentException thrown =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() ->
|
||||
new Registrar.Builder()
|
||||
.setLastExpiringFailoverCertNotificationSentDate(null)
|
||||
.build());
|
||||
assertThat(thrown)
|
||||
.hasMessageThat()
|
||||
.isEqualTo("Registrar lastExpiringFailoverCertNotificationSentDate cannot be null");
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_setAllowedTlds() {
|
||||
assertThat(
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
|
||||
import google.registry.persistence.transaction.JpaTestRules;
|
||||
import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension;
|
||||
import google.registry.testing.DatastoreEntityExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
import java.util.Optional;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
/** Tests for {@link google.registry.model.server.KmsSecretRevisionSqlDao}. */
|
||||
public class KmsSecretRevisionSqlDaoTest {
|
||||
|
||||
private final FakeClock fakeClock = new FakeClock();
|
||||
|
||||
@RegisterExtension
|
||||
@Order(value = 1)
|
||||
DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension();
|
||||
|
||||
@RegisterExtension
|
||||
JpaIntegrationWithCoverageExtension jpa =
|
||||
new JpaTestRules.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension();
|
||||
|
||||
@Test
|
||||
void testSaveAndRetrieve() {
|
||||
KmsSecretRevision revision = createRevision();
|
||||
jpaTm().transact(() -> KmsSecretRevisionSqlDao.save(revision));
|
||||
Optional<KmsSecretRevision> fromSql =
|
||||
jpaTm().transact(() -> KmsSecretRevisionSqlDao.getLatestRevision("secretName"));
|
||||
assertThat(fromSql.isPresent()).isTrue();
|
||||
assertAboutImmutableObjects().that(revision).isEqualExceptFields(fromSql.get(), "creationTime");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMultipleRevisions() {
|
||||
KmsSecretRevision revision = createRevision();
|
||||
jpaTm().transact(() -> KmsSecretRevisionSqlDao.save(revision));
|
||||
|
||||
KmsSecretRevision secondRevision = createRevision();
|
||||
secondRevision.encryptedValue = "someOtherValue";
|
||||
jpaTm().transact(() -> KmsSecretRevisionSqlDao.save(secondRevision));
|
||||
|
||||
Optional<KmsSecretRevision> fromSql =
|
||||
jpaTm().transact(() -> KmsSecretRevisionSqlDao.getLatestRevision("secretName"));
|
||||
assertThat(fromSql.isPresent()).isTrue();
|
||||
assertThat(fromSql.get().getEncryptedValue()).isEqualTo("someOtherValue");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNonexistent() {
|
||||
KmsSecretRevision revision = createRevision();
|
||||
jpaTm().transact(() -> KmsSecretRevisionSqlDao.save(revision));
|
||||
assertThat(
|
||||
jpaTm()
|
||||
.transact(() -> KmsSecretRevisionSqlDao.getLatestRevision("someOtherSecretName"))
|
||||
.isPresent())
|
||||
.isFalse();
|
||||
}
|
||||
|
||||
private KmsSecretRevision createRevision() {
|
||||
return new KmsSecretRevision.Builder()
|
||||
.setEncryptedValue("encrypted")
|
||||
.setKmsCryptoKeyVersionName("version")
|
||||
.setParent("secretName")
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
/** Unit tests for {@link google.registry.model.server.KmsSecretRevision}. */
|
||||
public class KmsSecretRevisionTest {
|
||||
|
||||
@RegisterExtension
|
||||
public final AppEngineExtension appEngine =
|
||||
AppEngineExtension.builder().withDatastoreAndCloudSql().build();
|
||||
|
||||
private KmsSecretRevision secretRevision;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
secretRevision =
|
||||
persistResource(
|
||||
new KmsSecretRevision.Builder()
|
||||
.setKmsCryptoKeyVersionName("foo")
|
||||
.setParent("bar")
|
||||
.setEncryptedValue("blah")
|
||||
.build());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_setEncryptedValue_tooLong_throwsException() {
|
||||
IllegalArgumentException thrown =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
() ->
|
||||
secretRevision =
|
||||
persistResource(
|
||||
new KmsSecretRevision.Builder()
|
||||
.setKmsCryptoKeyVersionName("foo")
|
||||
.setParent("bar")
|
||||
.setEncryptedValue(Strings.repeat("a", 64 * 1024 * 1024 + 1))
|
||||
.build()));
|
||||
assertThat(thrown).hasMessageThat().contains("Secret is greater than 67108864 bytes");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testPersistence() {
|
||||
assertThat(ofyTm().loadByEntity(secretRevision)).isEqualTo(secretRevision);
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.server;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
public class KmsSecretTest {
|
||||
|
||||
@RegisterExtension
|
||||
public final AppEngineExtension appEngine =
|
||||
AppEngineExtension.builder().withDatastoreAndCloudSql().build();
|
||||
|
||||
private KmsSecret secret;
|
||||
private KmsSecretRevision secretRevision;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
secretRevision =
|
||||
persistResource(
|
||||
new KmsSecretRevision.Builder()
|
||||
.setKmsCryptoKeyVersionName("foo")
|
||||
.setParent("bar")
|
||||
.setEncryptedValue("blah")
|
||||
.build());
|
||||
|
||||
secret = persistResource(KmsSecret.create("someSecret", secretRevision));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testPersistence() {
|
||||
assertThat(ofyTm().loadByEntity(secret)).isEqualTo(secret);
|
||||
}
|
||||
}
|
||||
+2
@@ -202,11 +202,13 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft
|
||||
JpaTransactionManagerImpl txnManager = new JpaTransactionManagerImpl(emf, clock);
|
||||
cachedTm = TransactionManagerFactory.jpaTm();
|
||||
TransactionManagerFactory.setJpaTm(Suppliers.ofInstance(txnManager));
|
||||
JpaTransactionManagerImpl.setReplaySqlToDatastoreOverrideForTest(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterEach(ExtensionContext context) {
|
||||
TransactionManagerFactory.setJpaTm(Suppliers.ofInstance(cachedTm));
|
||||
JpaTransactionManagerImpl.removeReplaySqlToDsOverrideForTest();
|
||||
cachedTm = null;
|
||||
}
|
||||
|
||||
|
||||
+18
-10
@@ -17,7 +17,7 @@ package google.registry.persistence.transaction;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.testing.DatabaseHelper.assertDetached;
|
||||
import static google.registry.testing.DatabaseHelper.assertDetachedFromEntityManager;
|
||||
import static google.registry.testing.TestDataHelper.fileClassPath;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
@@ -388,7 +388,8 @@ class JpaTransactionManagerImplTest {
|
||||
void load_succeeds() {
|
||||
assertThat(jpaTm().transact(() -> jpaTm().exists(theEntity))).isFalse();
|
||||
jpaTm().transact(() -> jpaTm().insert(theEntity));
|
||||
TestEntity persisted = jpaTm().transact(() -> assertDetached(jpaTm().loadByKey(theEntityKey)));
|
||||
TestEntity persisted =
|
||||
jpaTm().transact(() -> assertDetachedFromEntityManager(jpaTm().loadByKey(theEntityKey)));
|
||||
assertThat(persisted.name).isEqualTo("theEntity");
|
||||
assertThat(persisted.data).isEqualTo("foo");
|
||||
}
|
||||
@@ -404,7 +405,8 @@ class JpaTransactionManagerImplTest {
|
||||
@Test
|
||||
void loadByEntity_succeeds() {
|
||||
jpaTm().transact(() -> jpaTm().insert(theEntity));
|
||||
TestEntity persisted = jpaTm().transact(() -> assertDetached(jpaTm().loadByEntity(theEntity)));
|
||||
TestEntity persisted =
|
||||
jpaTm().transact(() -> assertDetachedFromEntityManager(jpaTm().loadByEntity(theEntity)));
|
||||
assertThat(persisted.name).isEqualTo("theEntity");
|
||||
assertThat(persisted.data).isEqualTo("foo");
|
||||
}
|
||||
@@ -414,7 +416,11 @@ class JpaTransactionManagerImplTest {
|
||||
assertThat(jpaTm().transact(() -> jpaTm().exists(theEntity))).isFalse();
|
||||
jpaTm().transact(() -> jpaTm().insert(theEntity));
|
||||
TestEntity persisted =
|
||||
jpaTm().transact(() -> assertDetached(jpaTm().loadByKeyIfPresent(theEntityKey).get()));
|
||||
jpaTm()
|
||||
.transact(
|
||||
() ->
|
||||
assertDetachedFromEntityManager(
|
||||
jpaTm().loadByKeyIfPresent(theEntityKey).get()));
|
||||
assertThat(persisted.name).isEqualTo("theEntity");
|
||||
assertThat(persisted.data).isEqualTo("foo");
|
||||
}
|
||||
@@ -431,7 +437,9 @@ class JpaTransactionManagerImplTest {
|
||||
assertThat(jpaTm().transact(() -> jpaTm().exists(compoundIdEntity))).isFalse();
|
||||
jpaTm().transact(() -> jpaTm().insert(compoundIdEntity));
|
||||
TestCompoundIdEntity persisted =
|
||||
jpaTm().transact(() -> assertDetached(jpaTm().loadByKey(compoundIdEntityKey)));
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> assertDetachedFromEntityManager(jpaTm().loadByKey(compoundIdEntityKey)));
|
||||
assertThat(persisted.name).isEqualTo("compoundIdEntity");
|
||||
assertThat(persisted.age).isEqualTo(10);
|
||||
assertThat(persisted.data).isEqualTo("foo");
|
||||
@@ -450,7 +458,7 @@ class JpaTransactionManagerImplTest {
|
||||
theEntityKey, VKey.createSql(TestEntity.class, "does-not-exist")));
|
||||
|
||||
assertThat(results).containsExactly(theEntityKey, theEntity);
|
||||
assertDetached(results.get(theEntityKey));
|
||||
assertDetachedFromEntityManager(results.get(theEntityKey));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -463,7 +471,7 @@ class JpaTransactionManagerImplTest {
|
||||
ImmutableMap<VKey<? extends TestEntity>, TestEntity> results =
|
||||
jpaTm().loadByKeysIfPresent(ImmutableList.of(theEntityKey));
|
||||
assertThat(results).containsExactly(theEntityKey, theEntity);
|
||||
assertDetached(results.get(theEntityKey));
|
||||
assertDetachedFromEntityManager(results.get(theEntityKey));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -478,7 +486,7 @@ class JpaTransactionManagerImplTest {
|
||||
.loadByEntitiesIfPresent(
|
||||
ImmutableList.of(theEntity, new TestEntity("does-not-exist", "bar")));
|
||||
assertThat(results).containsExactly(theEntity);
|
||||
assertDetached(results.get(0));
|
||||
assertDetachedFromEntityManager(results.get(0));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -491,7 +499,7 @@ class JpaTransactionManagerImplTest {
|
||||
ImmutableList<TestEntity> results =
|
||||
jpaTm().loadByEntities(ImmutableList.of(theEntity));
|
||||
assertThat(results).containsExactly(theEntity);
|
||||
assertDetached(results.get(0));
|
||||
assertDetachedFromEntityManager(results.get(0));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -503,7 +511,7 @@ class JpaTransactionManagerImplTest {
|
||||
.transact(
|
||||
() ->
|
||||
jpaTm().loadAllOf(TestEntity.class).stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(DatabaseHelper::assertDetachedFromEntityManager)
|
||||
.collect(toImmutableList()));
|
||||
assertThat(persisted).containsExactlyElementsIn(moreEntities);
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.GT, "bravo")
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(charlie);
|
||||
assertThat(
|
||||
@@ -86,7 +86,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.GTE, "charlie")
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(charlie);
|
||||
assertThat(
|
||||
@@ -95,7 +95,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.LT, "bravo")
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(alpha);
|
||||
assertThat(
|
||||
@@ -104,7 +104,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.LTE, "alpha")
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(alpha);
|
||||
}
|
||||
@@ -125,7 +125,7 @@ public class QueryComposerTest {
|
||||
assertThat(
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
DatabaseHelper.assertDetached(
|
||||
QueryComposerTest.assertDetachedIfJpa(
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.EQ, "alpha")
|
||||
.getSingleResult())))
|
||||
@@ -174,7 +174,7 @@ public class QueryComposerTest {
|
||||
.createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.GT, "alpha")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(bravo, charlie);
|
||||
assertThat(
|
||||
@@ -184,7 +184,7 @@ public class QueryComposerTest {
|
||||
.createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.GTE, "bravo")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(bravo, charlie);
|
||||
assertThat(
|
||||
@@ -194,7 +194,7 @@ public class QueryComposerTest {
|
||||
.createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.LT, "charlie")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(alpha, bravo);
|
||||
assertThat(
|
||||
@@ -204,7 +204,7 @@ public class QueryComposerTest {
|
||||
.createQueryComposer(TestEntity.class)
|
||||
.where("name", Comparator.LTE, "bravo")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(alpha, bravo);
|
||||
}
|
||||
@@ -228,7 +228,7 @@ public class QueryComposerTest {
|
||||
tm().createQueryComposer(TestEntity.class)
|
||||
.where("val", Comparator.EQ, 2)
|
||||
.first()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.get()))
|
||||
.isEqualTo(bravo);
|
||||
}
|
||||
@@ -243,7 +243,7 @@ public class QueryComposerTest {
|
||||
.where("val", Comparator.GT, 1)
|
||||
.orderBy("val")
|
||||
.stream()
|
||||
.map(DatabaseHelper::assertDetached)
|
||||
.map(QueryComposerTest::assertDetachedIfJpa)
|
||||
.collect(toImmutableList())))
|
||||
.containsExactly(bravo, alpha);
|
||||
}
|
||||
@@ -354,6 +354,13 @@ public class QueryComposerTest {
|
||||
.contains("The LIKE operation is not supported on Datastore.");
|
||||
}
|
||||
|
||||
private static <T> T assertDetachedIfJpa(T entity) {
|
||||
if (!tm().isOfy()) {
|
||||
return DatabaseHelper.assertDetachedFromEntityManager(entity);
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
@javax.persistence.Entity
|
||||
@Entity(name = "QueryComposerTestEntity")
|
||||
private static class TestEntity extends ImmutableObject {
|
||||
|
||||
@@ -22,36 +22,53 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.DatabaseHelper;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.InjectExtension;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.io.StreamCorruptedException;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
class TransactionTest {
|
||||
|
||||
private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ"));
|
||||
|
||||
@RegisterExtension
|
||||
final AppEngineExtension appEngine =
|
||||
AppEngineExtension.builder()
|
||||
.withDatastoreAndCloudSql()
|
||||
.withClock(fakeClock)
|
||||
.withOfyTestEntities(TestEntity.class)
|
||||
.withJpaUnitTestEntities(TestEntity.class)
|
||||
.build();
|
||||
|
||||
@RegisterExtension public final InjectExtension inject = new InjectExtension();
|
||||
|
||||
private TestEntity fooEntity, barEntity;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
JpaTransactionManagerImpl.removeReplaySqlToDsOverrideForTest();
|
||||
inject.setStaticField(Ofy.class, "clock", fakeClock);
|
||||
fooEntity = new TestEntity("foo");
|
||||
barEntity = new TestEntity("bar");
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
DatabaseHelper.removeDatabaseMigrationSchedule();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTransactionReplay() {
|
||||
Transaction txn = new Transaction.Builder().addUpdate(fooEntity).addUpdate(barEntity).build();
|
||||
@@ -102,14 +119,13 @@ class TransactionTest {
|
||||
|
||||
@Test
|
||||
void testTransactionSerialization() throws IOException {
|
||||
RegistryConfig.overrideCloudSqlReplicateTransactions(true);
|
||||
try {
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
jpaTm().insert(fooEntity);
|
||||
jpaTm().insert(barEntity);
|
||||
});
|
||||
DatabaseHelper.setMigrationScheduleToSqlPrimary(fakeClock);
|
||||
jpaTm()
|
||||
.transact(
|
||||
() -> {
|
||||
jpaTm().insert(fooEntity);
|
||||
jpaTm().insert(barEntity);
|
||||
});
|
||||
TransactionEntity txnEnt =
|
||||
jpaTm().transact(() -> jpaTm().loadByKey(VKey.createSql(TransactionEntity.class, 1L)));
|
||||
Transaction txn = Transaction.deserialize(txnEnt.getContents());
|
||||
@@ -125,9 +141,6 @@ class TransactionTest {
|
||||
assertThat(
|
||||
jpaTm().transact(() -> jpaTm().exists(VKey.createSql(TransactionEntity.class, 2L))))
|
||||
.isFalse();
|
||||
} finally {
|
||||
RegistryConfig.overrideCloudSqlReplicateTransactions(false);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -21,11 +21,11 @@ import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assumptions.assumeTrue;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.io.ByteSource;
|
||||
import com.google.common.io.CharStreams;
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.BouncyCastleProviderExtension;
|
||||
|
||||
@@ -40,11 +40,11 @@ import com.google.appengine.api.urlfetch.HTTPRequest;
|
||||
import com.google.appengine.api.urlfetch.HTTPResponse;
|
||||
import com.google.appengine.api.urlfetch.URLFetchService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteSource;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.rde.RdeRevision;
|
||||
import google.registry.model.registry.Registry;
|
||||
|
||||
@@ -35,12 +35,12 @@ import static java.util.Arrays.asList;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.net.InetAddresses;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
|
||||
@@ -29,9 +29,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.beam.rde.RdePipelineTest;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
@@ -120,7 +121,7 @@ class RdeStagingReducerTest {
|
||||
String outputFile = decryptGhostrydeGcsFile("soy_2000-01-01_thin_S1_R1.xml.ghostryde");
|
||||
assertThat(outputFile)
|
||||
.isEqualTo(
|
||||
readResourceUtf8(RdeStagingReducerTest.class, "reducer_brda.xml")
|
||||
readResourceUtf8(RdePipelineTest.class, "reducer_brda.xml")
|
||||
.replace("%RESEND%", " resend=\"1\""));
|
||||
compareLength(outputFile, "soy_2000-01-01_thin_S1_R1.xml.length");
|
||||
// BRDA doesn't write a report file.
|
||||
@@ -147,8 +148,7 @@ class RdeStagingReducerTest {
|
||||
String outputFile = decryptGhostrydeGcsFile("manual/soy_2000-01-01_thin_S1_R0.xml.ghostryde");
|
||||
assertThat(outputFile)
|
||||
.isEqualTo(
|
||||
readResourceUtf8(RdeStagingReducerTest.class, "reducer_brda.xml")
|
||||
.replace("%RESEND%", ""));
|
||||
readResourceUtf8(RdePipelineTest.class, "reducer_brda.xml").replace("%RESEND%", ""));
|
||||
compareLength(outputFile, "manual/soy_2000-01-01_thin_S1_R0.xml.length");
|
||||
// BRDA doesn't write a report file.
|
||||
assertThrows(
|
||||
@@ -169,12 +169,12 @@ class RdeStagingReducerTest {
|
||||
String outputFile = decryptGhostrydeGcsFile("soy_2000-01-01_full_S1_R1.xml.ghostryde");
|
||||
assertThat(outputFile)
|
||||
.isEqualTo(
|
||||
readResourceUtf8(RdeStagingReducerTest.class, "reducer_rde.xml")
|
||||
readResourceUtf8(RdePipelineTest.class, "reducer_rde.xml")
|
||||
.replace("%RESEND%", " resend=\"1\""));
|
||||
compareLength(outputFile, "soy_2000-01-01_full_S1_R1.xml.length");
|
||||
assertThat(decryptGhostrydeGcsFile("soy_2000-01-01_full_S1_R1-report.xml.ghostryde"))
|
||||
.isEqualTo(
|
||||
readResourceUtf8(RdeStagingReducerTest.class, "reducer_rde_report.xml")
|
||||
readResourceUtf8(RdePipelineTest.class, "reducer_rde_report.xml")
|
||||
.replace("%RESEND%", "1"));
|
||||
assertThat(loadCursorTime(CursorType.RDE_STAGING))
|
||||
.isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
|
||||
@@ -191,12 +191,11 @@ class RdeStagingReducerTest {
|
||||
String outputFile = decryptGhostrydeGcsFile("manual/soy_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
assertThat(outputFile)
|
||||
.isEqualTo(
|
||||
readResourceUtf8(RdeStagingReducerTest.class, "reducer_rde.xml")
|
||||
.replace("%RESEND%", ""));
|
||||
readResourceUtf8(RdePipelineTest.class, "reducer_rde.xml").replace("%RESEND%", ""));
|
||||
compareLength(outputFile, "manual/soy_2000-01-01_full_S1_R0.xml.length");
|
||||
assertThat(decryptGhostrydeGcsFile("manual/soy_2000-01-01_full_S1_R0-report.xml.ghostryde"))
|
||||
.isEqualTo(
|
||||
readResourceUtf8(RdeStagingReducerTest.class, "reducer_rde_report.xml")
|
||||
readResourceUtf8(RdePipelineTest.class, "reducer_rde_report.xml")
|
||||
.replace("%RESEND%", "0"));
|
||||
// No extra operations in manual mode.
|
||||
assertThat(loadCursorTime(CursorType.RDE_STAGING)).isEquivalentAccordingToCompareTo(now);
|
||||
|
||||
@@ -44,6 +44,7 @@ import static org.mockito.Mockito.when;
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.utils.SystemProperty;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.io.ByteSource;
|
||||
import com.google.common.io.CharStreams;
|
||||
import com.google.common.io.Files;
|
||||
@@ -51,7 +52,6 @@ import com.jcraft.jsch.JSch;
|
||||
import com.jcraft.jsch.JSchException;
|
||||
import com.jcraft.jsch.Session;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
|
||||
+1
-1
@@ -29,9 +29,9 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
|
||||
@@ -53,6 +53,7 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
|
||||
"billing_bucket",
|
||||
"REG-INV",
|
||||
true,
|
||||
"DATASTORE",
|
||||
new YearMonth(2017, 10),
|
||||
emailUtils,
|
||||
clock,
|
||||
@@ -82,6 +83,7 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
|
||||
"billing_bucket",
|
||||
"REG-INV",
|
||||
false,
|
||||
"DATASTORE",
|
||||
new YearMonth(2017, 10),
|
||||
emailUtils,
|
||||
clock,
|
||||
@@ -105,6 +107,7 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
|
||||
"billing_bucket",
|
||||
"REG-INV",
|
||||
false,
|
||||
"DATASTORE",
|
||||
new YearMonth(2017, 10),
|
||||
emailUtils,
|
||||
clock,
|
||||
|
||||
@@ -22,6 +22,7 @@ import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableTable;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
@@ -29,7 +30,6 @@ import google.registry.bigquery.BigqueryConnection;
|
||||
import google.registry.bigquery.BigqueryConnection.DestinationTable;
|
||||
import google.registry.bigquery.BigqueryUtils.TableType;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportType;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.FakeResponse;
|
||||
|
||||
+1
-1
@@ -29,9 +29,9 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.testing.TestLogHandler;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
import google.registry.model.registry.Registry;
|
||||
|
||||
@@ -30,7 +30,6 @@ import google.registry.model.registry.RegistryLockDaoTest;
|
||||
import google.registry.model.registry.RegistryTest;
|
||||
import google.registry.model.registry.label.ReservedListDaoTest;
|
||||
import google.registry.model.reporting.Spec11ThreatMatchTest;
|
||||
import google.registry.model.server.KmsSecretRevisionSqlDaoTest;
|
||||
import google.registry.model.server.LockTest;
|
||||
import google.registry.model.server.ServerSecretTest;
|
||||
import google.registry.model.smd.SignedMarkRevocationListDaoTest;
|
||||
@@ -89,7 +88,6 @@ import org.junit.runner.RunWith;
|
||||
DomainBaseSqlTest.class,
|
||||
DomainHistoryTest.class,
|
||||
HostHistoryTest.class,
|
||||
KmsSecretRevisionSqlDaoTest.class,
|
||||
LockTest.class,
|
||||
PollMessageTest.class,
|
||||
PremiumListDaoTest.class,
|
||||
|
||||
+5
-24
@@ -26,15 +26,16 @@ import com.google.common.testing.TestLogHandler;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.annotation.Entity;
|
||||
import com.googlecode.objectify.annotation.Id;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.persistence.transaction.JpaTransactionManagerImpl;
|
||||
import google.registry.persistence.transaction.TransactionEntity;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.DatabaseHelper;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.InjectExtension;
|
||||
import java.util.List;
|
||||
@@ -68,41 +69,21 @@ public class ReplicateToDatastoreActionTest {
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() {
|
||||
JpaTransactionManagerImpl.removeReplaySqlToDsOverrideForTest();
|
||||
injectExtension.setStaticField(Ofy.class, "clock", fakeClock);
|
||||
// Use a single bucket to expose timestamp inversion problems.
|
||||
injectExtension.setStaticField(
|
||||
CommitLogBucket.class, "bucketIdSupplier", Suppliers.ofInstance(1));
|
||||
fakeClock.setAutoIncrementByOneMilli();
|
||||
RegistryConfig.overrideCloudSqlReplicateTransactions(true);
|
||||
DatabaseHelper.setMigrationScheduleToSqlPrimary(fakeClock);
|
||||
Logger.getLogger(ReplicateToDatastoreAction.class.getCanonicalName()).addHandler(logHandler);
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
ofyTm()
|
||||
.transact(
|
||||
() ->
|
||||
DatabaseMigrationStateSchedule.set(
|
||||
ImmutableSortedMap.of(
|
||||
START_OF_TIME,
|
||||
MigrationState.DATASTORE_ONLY,
|
||||
now,
|
||||
MigrationState.DATASTORE_PRIMARY,
|
||||
now.plusHours(1),
|
||||
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
|
||||
now.plusHours(2),
|
||||
MigrationState.SQL_PRIMARY)));
|
||||
fakeClock.advanceBy(Duration.standardDays(1));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void tearDown() {
|
||||
DatabaseHelper.removeDatabaseMigrationSchedule();
|
||||
fakeClock.disableAutoIncrement();
|
||||
RegistryConfig.overrideCloudSqlReplicateTransactions(false);
|
||||
ofyTm()
|
||||
.transact(
|
||||
() ->
|
||||
ofyTm()
|
||||
.loadSingleton(DatabaseMigrationStateSchedule.class)
|
||||
.ifPresent(ofyTm()::delete));
|
||||
DatabaseMigrationStateSchedule.CACHE.invalidateAll();
|
||||
}
|
||||
|
||||
@RetryingTest(4)
|
||||
|
||||
@@ -35,6 +35,7 @@ import static google.registry.model.ResourceTransferUtils.createTransferResponse
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.model.registry.Registry.TldState.GENERAL_AVAILABILITY;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.ofyTmOrDoNothing;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
@@ -71,6 +72,8 @@ import google.registry.model.EppResourceUtils;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.contact.ContactAuthInfo;
|
||||
import google.registry.model.contact.ContactHistory;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
@@ -123,6 +126,7 @@ import org.joda.money.CurrencyUnit;
|
||||
import org.joda.money.Money;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/** Static utils for setting up test resources. */
|
||||
public class DatabaseHelper {
|
||||
@@ -1351,16 +1355,54 @@ public class DatabaseHelper {
|
||||
}
|
||||
|
||||
/**
|
||||
* In JPA mode, assert that the given entity is detached from the current entity manager.
|
||||
* Asserts that the given entity is detached from the current JPA entity manager.
|
||||
*
|
||||
* <p>Returns the original entity object.
|
||||
*/
|
||||
public static <T> T assertDetached(T entity) {
|
||||
if (!tm().isOfy()) {
|
||||
assertThat(jpaTm().getEntityManager().contains(entity)).isFalse();
|
||||
}
|
||||
public static <T> T assertDetachedFromEntityManager(T entity) {
|
||||
assertThat(jpaTm().getEntityManager().contains(entity)).isFalse();
|
||||
return entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a SQL_PRIMARY state on the {@link DatabaseMigrationStateSchedule}.
|
||||
*
|
||||
* <p>In order to allow for tests to manipulate the clock how they need, we start the transitions
|
||||
* one millisecond after the clock's current time (in case the clock's current value is
|
||||
* START_OF_TIME). We then advance the clock one second so that we're in the SQL_PRIMARY phase.
|
||||
*
|
||||
* <p>We must use the current time, otherwise the setting of the migration state will fail due to
|
||||
* an invalid transition.
|
||||
*/
|
||||
public static void setMigrationScheduleToSqlPrimary(FakeClock fakeClock) {
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
ofyTm()
|
||||
.transact(
|
||||
() ->
|
||||
DatabaseMigrationStateSchedule.set(
|
||||
ImmutableSortedMap.of(
|
||||
START_OF_TIME,
|
||||
MigrationState.DATASTORE_ONLY,
|
||||
now.plusMillis(1),
|
||||
MigrationState.DATASTORE_PRIMARY,
|
||||
now.plusMillis(2),
|
||||
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
|
||||
now.plusMillis(3),
|
||||
MigrationState.SQL_PRIMARY)));
|
||||
fakeClock.advanceBy(Duration.standardSeconds(1));
|
||||
}
|
||||
|
||||
/** Removes the database migration schedule, in essence transitioning to DATASTORE_ONLY. */
|
||||
public static void removeDatabaseMigrationSchedule() {
|
||||
// use the raw calls because going SQL_PRIMARY -> DATASTORE_ONLY is not valid
|
||||
ofyTm()
|
||||
.transact(
|
||||
() ->
|
||||
ofyTm()
|
||||
.loadSingleton(DatabaseMigrationStateSchedule.class)
|
||||
.ifPresent(ofyTm()::delete));
|
||||
DatabaseMigrationStateSchedule.CACHE.invalidateAll();
|
||||
}
|
||||
|
||||
private DatabaseHelper() {}
|
||||
}
|
||||
|
||||
@@ -22,12 +22,12 @@ import com.google.common.base.Suppliers;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.ReplayQueue;
|
||||
import google.registry.model.ofy.TransactionInfo;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.persistence.transaction.JpaTransactionManagerImpl;
|
||||
import google.registry.persistence.transaction.TransactionEntity;
|
||||
import google.registry.schema.replay.DatastoreEntity;
|
||||
import google.registry.schema.replay.ReplicateToDatastoreAction;
|
||||
@@ -92,7 +92,7 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
|
||||
// transaction manager gets injected.
|
||||
inOfyContext = DualDatabaseTestInvocationContextProvider.inOfyContext(context);
|
||||
if (sqlToDsReplicator != null && !inOfyContext) {
|
||||
RegistryConfig.overrideCloudSqlReplicateTransactions(true);
|
||||
JpaTransactionManagerImpl.setReplaySqlToDatastoreOverrideForTest(true);
|
||||
}
|
||||
|
||||
context.getStore(ExtensionContext.Namespace.GLOBAL).put(ReplayExtension.class, this);
|
||||
@@ -105,7 +105,7 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
|
||||
replay();
|
||||
injectExtension.afterEach(context);
|
||||
if (sqlToDsReplicator != null) {
|
||||
RegistryConfig.overrideCloudSqlReplicateTransactions(false);
|
||||
JpaTransactionManagerImpl.setReplaySqlToDatastoreOverrideForTest(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,7 +116,6 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
|
||||
"PremiumListEntry",
|
||||
"ReservedList",
|
||||
"RdeRevision",
|
||||
"KmsSecretRevision",
|
||||
"ServerSecret",
|
||||
"SignedMarkRevocationList",
|
||||
"ClaimsListShard",
|
||||
|
||||
@@ -75,7 +75,8 @@ public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity
|
||||
beforeSqlDeleteCallCount++;
|
||||
}
|
||||
|
||||
public static void beforeSqlSave(TestObject testObject) {
|
||||
@Override
|
||||
public void beforeSqlSaveOnReplay() {
|
||||
beforeSqlSaveCallCount++;
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ class CreateRegistrarCommandTest extends CommandTestCase<CreateRegistrarCommand>
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
fakeClock);
|
||||
|
||||
@@ -21,6 +21,7 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.testing.DatabaseHelper;
|
||||
import google.registry.testing.DualDatabaseTest;
|
||||
import google.registry.testing.TestOfyAndSql;
|
||||
import org.joda.time.DateTime;
|
||||
@@ -33,7 +34,7 @@ public class GetDatabaseMigrationStateCommandTest
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
ofyTm().transact(() -> DatabaseMigrationStateSchedule.set(DEFAULT_TRANSITION_MAP.toValueMap()));
|
||||
DatabaseHelper.removeDatabaseMigrationSchedule();
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
|
||||
@@ -25,32 +25,20 @@ import com.beust.jcommander.ParameterException;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.testing.DatabaseHelper;
|
||||
import google.registry.testing.DualDatabaseTest;
|
||||
import google.registry.testing.TestOfyAndSql;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
|
||||
/** Tests for {@link SetDatabaseMigrationStateCommand}. */
|
||||
@DualDatabaseTest
|
||||
public class SetDatabaseMigrationStateCommandTest
|
||||
extends CommandTestCase<SetDatabaseMigrationStateCommand> {
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
// clear out any static state that may have been persisted
|
||||
ofyTm()
|
||||
.transact(
|
||||
() ->
|
||||
ofyTm()
|
||||
.loadSingleton(DatabaseMigrationStateSchedule.class)
|
||||
.ifPresent(ofyTm()::delete));
|
||||
DatabaseMigrationStateSchedule.CACHE.invalidateAll();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void afterEach() {
|
||||
ofyTm().transact(() -> DatabaseMigrationStateSchedule.set(DEFAULT_TRANSITION_MAP.toValueMap()));
|
||||
DatabaseHelper.removeDatabaseMigrationSchedule();
|
||||
}
|
||||
|
||||
@TestOfyAndSql
|
||||
|
||||
@@ -53,6 +53,7 @@ class UpdateRegistrarCommandTest extends CommandTestCase<UpdateRegistrarCommand>
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
fakeClock);
|
||||
|
||||
@@ -65,6 +65,7 @@ class ValidateLoginCredentialsCommandTest extends CommandTestCase<ValidateLoginC
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
fakeClock);
|
||||
|
||||
@@ -27,12 +27,12 @@ import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.Duration.standardDays;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.domain.secdns.DelegationSignerData;
|
||||
import google.registry.model.eppcommon.StatusValue;
|
||||
import google.registry.model.host.HostResource;
|
||||
|
||||
+1
@@ -128,6 +128,7 @@ public abstract class RegistrarSettingsActionTestCase {
|
||||
new CertificateChecker(
|
||||
ImmutableSortedMap.of(START_OF_TIME, 825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
|
||||
30,
|
||||
15,
|
||||
2048,
|
||||
ImmutableSet.of("secp256r1", "secp384r1"),
|
||||
clock);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user