mirror of
https://github.com/google/nomulus
synced 2026-08-14 11:16:14 +00:00
Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  gbrodmanandGitHub
|
9a2fb6f8b4 | ||
|
gbrodmanandGitHub
|
abc240fc2d | ||
 Michael MullerandGitHub
|
a424030a65 | ||
|
gbrodmanandGitHub
|
72abc824d5 | ||
 Lai JiangandGitHub
|
b2ec088749 | ||
 Ben McIlwainandGitHub
|
6d40fe41e6 | ||
 Weimin YuandGitHub
|
7dafbf6ae1 | ||
|
Michael MullerandGitHub
|
9b8f10c595 | ||
|
Lai JiangandGitHub
|
18d51738ce | ||
|
Lai JiangandGitHub
|
76d63b24a8 | ||
|
Michael MullerandGitHub
|
eb1b283ba3 | ||
|
Lai JiangandGitHub
|
63e4f4f10a | ||
 sarahcaseybotandGitHub
|
2c3279ba95 | ||
|
Ben McIlwainandGitHub
|
89925f9ff2 | ||
|
Lai JiangandGitHub
|
585765b83a | ||
|
Ben McIlwainandGitHub
|
cddcfc49ed | ||
|
sarahcaseybotandGitHub
|
fb7558121b | ||
|
Lai JiangandGitHub
|
1719d066cf | ||
|
gbrodmanandGitHub
|
fa1b34b020 | ||
|
Lai JiangandGitHub
|
4298084406 | ||
|
Weimin YuandGitHub
|
9b5f1756f1 | ||
|
Lai JiangandGitHub
|
62236f7581 | ||
|
Lai JiangandGitHub
|
e1ea176daa | ||
|
gbrodmanandGitHub
|
bb9b669014 | ||
|
gbrodmanandGitHub
|
dddeb48356 | ||
|
Lai JiangandGitHub
|
c878679770 |
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -11,13 +11,13 @@ com.google.api.grpc:proto-google-iam-v1:1.3.4
|
||||
com.google.api:api-common:2.2.0
|
||||
com.google.api:gax-httpjson:0.103.1
|
||||
com.google.api:gax:2.18.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.auth:google-auth-library-credentials:1.7.0
|
||||
com.google.auth:google-auth-library-oauth2-http:1.7.0
|
||||
com.google.auto.value:auto-value-annotations:1.9
|
||||
com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
||||
@@ -267,6 +267,12 @@
|
||||
"moduleLicense": "Public Domain",
|
||||
"moduleName": "org.tukaani:xz"
|
||||
},
|
||||
{
|
||||
// "Apache License, Version 2.0".
|
||||
"moduleLicense": null,
|
||||
"moduleVersion": "2.10.0",
|
||||
"moduleName": "com.google.gwt:gwt-user"
|
||||
},
|
||||
{
|
||||
// "Apache License, Version 2.0". The plugin is able to parse up to
|
||||
// 2.11.3 correctly but then something changed with 2.12.* and it no
|
||||
|
||||
@@ -196,7 +196,6 @@ dependencies {
|
||||
compile deps['com.google.apis:google-api-services-storage']
|
||||
testCompile deps['com.google.appengine:appengine-api-stubs']
|
||||
compile deps['com.google.appengine.tools:appengine-gcs-client']
|
||||
compile deps['com.google.appengine.tools:appengine-mapreduce']
|
||||
compile deps['com.google.appengine.tools:appengine-pipeline']
|
||||
compile deps['com.google.appengine:appengine-remote-api']
|
||||
compile deps['com.google.auth:google-auth-library-credentials']
|
||||
@@ -703,24 +702,9 @@ createToolTask(
|
||||
'google.registry.tools.DevTool',
|
||||
sourceSets.nonprod)
|
||||
|
||||
createToolTask(
|
||||
'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
|
||||
|
||||
createToolTask(
|
||||
'validateDatabasePipeline', 'google.registry.beam.comparedb.ValidateDatabasePipeline')
|
||||
|
||||
|
||||
createToolTask(
|
||||
'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
|
||||
|
||||
// Caller must provide projectId, GCP region, runner, and the kinds to delete
|
||||
// (comma-separated kind names or '*' for all). E.g.:
|
||||
// nom_build :core:bulkDeleteDatastore --args="--project=domain-registry-crash \
|
||||
// --region=us-central1 --runner=DataflowRunner --kindsToDelete=*"
|
||||
createToolTask(
|
||||
'bulkDeleteDatastore',
|
||||
'google.registry.beam.datastore.BulkDeleteDatastorePipeline')
|
||||
|
||||
project.tasks.create('generateSqlSchema', JavaExec) {
|
||||
classpath = sourceSets.nonprod.runtimeClasspath
|
||||
main = 'google.registry.tools.DevTool'
|
||||
@@ -765,11 +749,6 @@ createUberJar(
|
||||
// User should install gcloud and login to GCP before invoking this tasks.
|
||||
if (environment == 'alpha') {
|
||||
def pipelines = [
|
||||
bulkDeleteDatastore:
|
||||
[
|
||||
mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
|
||||
metaData : 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
|
||||
],
|
||||
spec11 :
|
||||
[
|
||||
mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -103,7 +101,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -115,7 +113,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -123,7 +121,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -133,28 +131,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -211,7 +208,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -263,7 +259,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -280,13 +275,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:slf4j-api:1.7.36
|
||||
org.springframework:spring-core:5.3.18
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -27,11 +26,11 @@ com.github.jnr:jnr-ffi:2.2.11
|
||||
com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -84,10 +83,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -102,7 +100,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -114,7 +112,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -122,7 +120,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -132,28 +130,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -206,7 +203,6 @@ io.opencensus:opencensus-exporter-stats-stackdriver:0.31.0
|
||||
io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -256,7 +252,6 @@ org.checkerframework:checker-qual:3.22.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -272,13 +267,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:slf4j-api:1.7.36
|
||||
org.springframework:spring-core:5.3.18
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -103,8 +101,8 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -116,7 +114,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -124,7 +122,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -134,28 +132,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -219,7 +216,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -273,7 +269,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -290,13 +285,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:jcl-over-slf4j:1.7.30
|
||||
org.slf4j:jul-to-slf4j:1.7.30
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -103,8 +101,8 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -116,7 +114,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -124,7 +122,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -134,28 +132,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -219,7 +216,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -272,7 +268,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -288,13 +283,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:jcl-over-slf4j:1.7.30
|
||||
org.slf4j:jul-to-slf4j:1.7.30
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
com.sun.activation:jakarta.activation:1.2.2
|
||||
com.sun.activation:javax.activation:1.2.0
|
||||
com.sun.xml.bind:jaxb-impl:2.3.3
|
||||
com.sun.xml.bind:jaxb-osgi:4.0.0
|
||||
com.sun.xml.bind:jaxb-osgi:4.0.0-M4
|
||||
com.sun.xml.bind:jaxb-xjc:2.3.3
|
||||
jakarta.activation:jakarta.activation-api:2.1.0
|
||||
jakarta.xml.bind:jakarta.xml.bind-api:4.0.0
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -103,7 +101,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -115,7 +113,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -123,7 +121,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -133,28 +131,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -211,7 +208,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -263,7 +259,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -280,13 +275,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:slf4j-api:1.7.36
|
||||
org.springframework:spring-core:5.3.18
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -27,11 +26,11 @@ com.github.jnr:jnr-ffi:2.2.11
|
||||
com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -84,10 +83,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -102,7 +100,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -114,7 +112,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -122,7 +120,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -132,28 +130,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -206,7 +203,6 @@ io.opencensus:opencensus-exporter-stats-stackdriver:0.31.0
|
||||
io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -257,7 +253,6 @@ org.checkerframework:checker-qual:3.22.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -274,13 +269,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:slf4j-api:1.7.36
|
||||
org.springframework:spring-core:5.3.18
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -103,7 +101,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -115,7 +113,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -123,7 +121,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -133,28 +131,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -218,7 +215,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -272,7 +268,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -289,13 +284,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:jcl-over-slf4j:1.7.30
|
||||
org.slf4j:jul-to-slf4j:1.7.30
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -103,7 +101,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -115,7 +113,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -123,7 +121,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -133,28 +131,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -218,7 +215,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -272,7 +268,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -289,13 +284,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:jcl-over-slf4j:1.7.30
|
||||
org.slf4j:jul-to-slf4j:1.7.30
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -103,7 +101,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -115,7 +113,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -123,7 +121,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -133,28 +131,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -218,7 +215,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -272,7 +268,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -289,13 +284,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:jcl-over-slf4j:1.7.30
|
||||
org.slf4j:jul-to-slf4j:1.7.30
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-remote-api:2.0.5
|
||||
@@ -103,8 +101,8 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -116,7 +114,7 @@ com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -124,7 +122,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -134,28 +132,27 @@ com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -219,7 +216,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -272,7 +268,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -288,13 +283,12 @@ org.joda:joda-money:1.0.1
|
||||
org.json:json:20200518
|
||||
org.jsoup:jsoup:1.15.1
|
||||
org.jvnet.staxex:stax-ex:1.8
|
||||
org.objenesis:objenesis:1.2
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.slf4j:jcl-over-slf4j:1.7.30
|
||||
org.slf4j:jul-to-slf4j:1.7.30
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-api-stubs:2.0.5
|
||||
@@ -104,7 +102,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -112,12 +110,12 @@ com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-firestore:3.0.14
|
||||
com.google.cloud:google-cloud-monitoring:1.82.0
|
||||
com.google.cloud:google-cloud-nio:0.124.4
|
||||
com.google.cloud:google-cloud-nio:0.124.2
|
||||
com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -125,7 +123,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -136,12 +134,12 @@ com.google.guava:guava-testlib:31.1-jre
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -149,18 +147,17 @@ com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:contrib:1.0.7
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.3
|
||||
com.google.truth:truth:1.1.3
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -220,7 +217,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -273,7 +269,7 @@ org.apache.mina:mina-core:2.1.6
|
||||
org.apache.sshd:sshd-core:2.0.0
|
||||
org.apache.sshd:sshd-scp:2.0.0
|
||||
org.apache.sshd:sshd-sftp:2.0.0
|
||||
org.apache.tomcat:tomcat-annotations-api:10.1.0-M16
|
||||
org.apache.tomcat:tomcat-annotations-api:10.1.0-M15
|
||||
org.bouncycastle:bcpg-jdk15on:1.67
|
||||
org.bouncycastle:bcpkix-jdk15on:1.67
|
||||
org.bouncycastle:bcprov-jdk15on:1.67
|
||||
@@ -283,7 +279,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -324,7 +319,7 @@ org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.seleniumhq.selenium:selenium-api:3.141.59
|
||||
org.seleniumhq.selenium:selenium-chrome-driver:3.141.59
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -27,11 +26,11 @@ com.github.jnr:jnr-ffi:2.2.11
|
||||
com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -84,10 +83,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-api-stubs:2.0.5
|
||||
@@ -103,7 +101,7 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -111,12 +109,12 @@ com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-firestore:3.0.14
|
||||
com.google.cloud:google-cloud-monitoring:1.82.0
|
||||
com.google.cloud:google-cloud-nio:0.124.4
|
||||
com.google.cloud:google-cloud-nio:0.124.2
|
||||
com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -124,7 +122,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -135,12 +133,12 @@ com.google.guava:guava-testlib:31.1-jre
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -148,18 +146,17 @@ com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:contrib:1.0.7
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.3
|
||||
com.google.truth:truth:1.1.3
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -215,7 +212,6 @@ io.opencensus:opencensus-exporter-stats-stackdriver:0.31.0
|
||||
io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -268,7 +264,7 @@ org.apache.mina:mina-core:2.1.6
|
||||
org.apache.sshd:sshd-core:2.0.0
|
||||
org.apache.sshd:sshd-scp:2.0.0
|
||||
org.apache.sshd:sshd-sftp:2.0.0
|
||||
org.apache.tomcat:tomcat-annotations-api:10.1.0-M16
|
||||
org.apache.tomcat:tomcat-annotations-api:10.1.0-M15
|
||||
org.apiguardian:apiguardian-api:1.1.2
|
||||
org.bouncycastle:bcpg-jdk15on:1.67
|
||||
org.bouncycastle:bcpkix-jdk15on:1.67
|
||||
@@ -278,7 +274,6 @@ org.checkerframework:checker-qual:3.22.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -311,14 +306,13 @@ org.mockito:mockito-core:4.6.1
|
||||
org.mockito:mockito-junit-jupiter:4.6.1
|
||||
org.mortbay.jetty:jetty-util:6.1.26
|
||||
org.mortbay.jetty:jetty:6.1.26
|
||||
org.objenesis:objenesis:1.2
|
||||
org.opentest4j:opentest4j:1.2.0
|
||||
org.ow2.asm:asm-analysis:9.3
|
||||
org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.seleniumhq.selenium:selenium-api:3.141.59
|
||||
org.seleniumhq.selenium:selenium-chrome-driver:3.141.59
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-api-stubs:2.0.5
|
||||
@@ -104,8 +102,8 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -113,12 +111,12 @@ com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-firestore:3.0.14
|
||||
com.google.cloud:google-cloud-monitoring:1.82.0
|
||||
com.google.cloud:google-cloud-nio:0.124.4
|
||||
com.google.cloud:google-cloud-nio:0.124.2
|
||||
com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -126,7 +124,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -137,12 +135,12 @@ com.google.guava:guava-testlib:31.1-jre
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -150,18 +148,17 @@ com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:contrib:1.0.7
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.3
|
||||
com.google.truth:truth:1.1.3
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -229,7 +226,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -283,7 +279,7 @@ org.apache.mina:mina-core:2.1.6
|
||||
org.apache.sshd:sshd-core:2.0.0
|
||||
org.apache.sshd:sshd-scp:2.0.0
|
||||
org.apache.sshd:sshd-sftp:2.0.0
|
||||
org.apache.tomcat:tomcat-annotations-api:10.1.0-M16
|
||||
org.apache.tomcat:tomcat-annotations-api:10.1.0-M15
|
||||
org.bouncycastle:bcpg-jdk15on:1.67
|
||||
org.bouncycastle:bcpkix-jdk15on:1.67
|
||||
org.bouncycastle:bcprov-jdk15on:1.67
|
||||
@@ -293,7 +289,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -334,7 +329,7 @@ org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.seleniumhq.selenium:selenium-api:3.141.59
|
||||
org.seleniumhq.selenium:selenium-chrome-driver:3.141.59
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
antlr:antlr:2.7.7
|
||||
aopalliance:aopalliance:1.0
|
||||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.101tec:zkclient:0.10
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.13.3
|
||||
@@ -28,11 +27,11 @@ com.github.jnr:jnr-posix:3.1.15
|
||||
com.github.jnr:jnr-unixsocket:0.38.17
|
||||
com.github.jnr:jnr-x86asm:1.0.2
|
||||
com.google.android:annotations:4.1.1.4
|
||||
com.google.api-client:google-api-client-appengine:1.35.1
|
||||
com.google.api-client:google-api-client-appengine:1.35.0
|
||||
com.google.api-client:google-api-client-jackson2:1.32.2
|
||||
com.google.api-client:google-api-client-java6:1.35.1
|
||||
com.google.api-client:google-api-client-servlet:1.35.1
|
||||
com.google.api-client:google-api-client:1.35.1
|
||||
com.google.api-client:google-api-client-java6:1.35.0
|
||||
com.google.api-client:google-api-client-servlet:1.35.0
|
||||
com.google.api-client:google-api-client:1.35.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.10.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.134.0
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.134.0
|
||||
@@ -85,10 +84,9 @@ com.google.apis:google-api-services-iamcredentials:v1-rev20210326-1.32.1
|
||||
com.google.apis:google-api-services-monitoring:v3-rev20220525-1.32.1
|
||||
com.google.apis:google-api-services-pubsub:v1-rev20211130-1.32.1
|
||||
com.google.apis:google-api-services-sheets:v4-rev20220411-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220513-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220604-1.32.1
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20220323-1.32.1
|
||||
com.google.apis:google-api-services-storage:v1-rev20220509-1.32.1
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
com.google.appengine:appengine-api-1.0-sdk:2.0.5
|
||||
com.google.appengine:appengine-api-stubs:2.0.5
|
||||
@@ -104,8 +102,8 @@ com.google.cloud.bigdataoss:util:2.2.6
|
||||
com.google.cloud.bigtable:bigtable-client-core:1.26.3
|
||||
com.google.cloud.bigtable:bigtable-metrics-api:1.26.3
|
||||
com.google.cloud.datastore:datastore-v1-proto-client:2.1.3
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.1
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.1
|
||||
com.google.cloud.sql:jdbc-socket-factory-core:1.6.0
|
||||
com.google.cloud.sql:postgres-socket-factory:1.6.0
|
||||
com.google.cloud:google-cloud-bigquerystorage:2.10.0
|
||||
com.google.cloud:google-cloud-bigtable:2.5.3
|
||||
com.google.cloud:google-cloud-core-grpc:2.4.0
|
||||
@@ -113,12 +111,12 @@ com.google.cloud:google-cloud-core-http:2.7.1
|
||||
com.google.cloud:google-cloud-core:2.7.1
|
||||
com.google.cloud:google-cloud-firestore:3.0.14
|
||||
com.google.cloud:google-cloud-monitoring:1.82.0
|
||||
com.google.cloud:google-cloud-nio:0.124.4
|
||||
com.google.cloud:google-cloud-nio:0.124.2
|
||||
com.google.cloud:google-cloud-pubsub:1.116.0
|
||||
com.google.cloud:google-cloud-pubsublite:1.5.0
|
||||
com.google.cloud:google-cloud-secretmanager:2.2.0
|
||||
com.google.cloud:google-cloud-spanner:6.20.0
|
||||
com.google.cloud:google-cloud-storage:2.8.0
|
||||
com.google.cloud:google-cloud-storage:2.7.2
|
||||
com.google.cloud:google-cloud-tasks:2.2.0
|
||||
com.google.cloud:grpc-gcp:1.1.0
|
||||
com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.14
|
||||
@@ -126,7 +124,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.9.0
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.42
|
||||
com.google.errorprone:error_prone_annotations:2.14.0
|
||||
com.google.errorprone:error_prone_annotations:2.13.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flatbuffers:flatbuffers-java:1.12.0
|
||||
com.google.flogger:flogger-system-backend:0.7.4
|
||||
@@ -137,12 +135,12 @@ com.google.guava:guava-testlib:31.1-jre
|
||||
com.google.guava:guava:31.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.42.0
|
||||
com.google.http-client:google-http-client-appengine:1.42.0
|
||||
com.google.http-client:google-http-client-gson:1.42.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.41.8
|
||||
com.google.http-client:google-http-client-appengine:1.41.8
|
||||
com.google.http-client:google-http-client-gson:1.41.8
|
||||
com.google.http-client:google-http-client-jackson2:1.41.8
|
||||
com.google.http-client:google-http-client-protobuf:1.40.1
|
||||
com.google.http-client:google-http-client:1.42.0
|
||||
com.google.http-client:google-http-client:1.41.8
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
@@ -150,18 +148,17 @@ com.google.jsinterop:jsinterop-annotations:2.0.0
|
||||
com.google.monitoring-client:contrib:1.0.7
|
||||
com.google.monitoring-client:metrics:1.0.7
|
||||
com.google.monitoring-client:stackdriver:1.0.7
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.1
|
||||
com.google.oauth-client:google-oauth-client:1.34.1
|
||||
com.google.oauth-client:google-oauth-client-appengine:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-java6:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.34.0
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.34.0
|
||||
com.google.oauth-client:google-oauth-client:1.34.0
|
||||
com.google.protobuf:protobuf-java-util:3.20.1
|
||||
com.google.protobuf:protobuf-java:3.20.1
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.3
|
||||
com.google.truth:truth:1.1.3
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
com.googlecode.json-simple:json-simple:1.1.1
|
||||
com.ibm.icu:icu4j:71.1
|
||||
com.jcraft:jsch:0.1.55
|
||||
@@ -229,7 +226,6 @@ io.opencensus:opencensus-impl-core:0.31.0
|
||||
io.opencensus:opencensus-impl:0.31.0
|
||||
io.opencensus:opencensus-proto:0.2.0
|
||||
io.perfmark:perfmark-api:0.25.0
|
||||
it.unimi.dsi:fastutil:6.5.16
|
||||
javax.activation:activation:1.1
|
||||
javax.activation:javax.activation-api:1.2.0
|
||||
javax.annotation:javax.annotation-api:1.3.2
|
||||
@@ -283,7 +279,7 @@ org.apache.mina:mina-core:2.1.6
|
||||
org.apache.sshd:sshd-core:2.0.0
|
||||
org.apache.sshd:sshd-scp:2.0.0
|
||||
org.apache.sshd:sshd-sftp:2.0.0
|
||||
org.apache.tomcat:tomcat-annotations-api:10.1.0-M16
|
||||
org.apache.tomcat:tomcat-annotations-api:10.1.0-M15
|
||||
org.bouncycastle:bcpg-jdk15on:1.67
|
||||
org.bouncycastle:bcpkix-jdk15on:1.67
|
||||
org.bouncycastle:bcprov-jdk15on:1.67
|
||||
@@ -293,7 +289,6 @@ org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.21
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
org.easymock:easymock:3.0
|
||||
org.flywaydb:flyway-core:8.5.12
|
||||
org.glassfish.jaxb:jaxb-runtime:2.3.1
|
||||
org.glassfish.jaxb:txw2:2.3.1
|
||||
@@ -334,7 +329,7 @@ org.ow2.asm:asm-commons:9.2
|
||||
org.ow2.asm:asm-tree:9.3
|
||||
org.ow2.asm:asm-util:9.3
|
||||
org.ow2.asm:asm:9.3
|
||||
org.postgresql:postgresql:42.4.0
|
||||
org.postgresql:postgresql:42.3.6
|
||||
org.rnorth.duct-tape:duct-tape:1.0.8
|
||||
org.seleniumhq.selenium:selenium-api:3.141.59
|
||||
org.seleniumhq.selenium:selenium-chrome-driver:3.141.59
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.appengine.api.ThreadManager.currentRequestThreadFactory;
|
||||
import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.BUCKET_OVERRIDE_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.FROM_TIME_PARAM;
|
||||
import static google.registry.backup.RestoreCommitLogsAction.TO_TIME_PARAM;
|
||||
import static google.registry.request.RequestParameters.extractOptionalParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static java.util.concurrent.Executors.newFixedThreadPool;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.google.common.util.concurrent.ListeningExecutorService;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.cron.CommitLogFanoutAction;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.Parameter;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.inject.Qualifier;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* Dagger module for backup package.
|
||||
*
|
||||
* @see "google.registry.module.backend.BackendComponent"
|
||||
*/
|
||||
@Module
|
||||
public final class BackupModule {
|
||||
|
||||
/** Dagger qualifier for backups. */
|
||||
@Qualifier
|
||||
@Documented
|
||||
public @interface Backups {}
|
||||
|
||||
/** Number of threads in the threaded executor. */
|
||||
private static final int NUM_THREADS = 10;
|
||||
|
||||
@Provides
|
||||
@Parameter("bucket")
|
||||
static int provideBucket(HttpServletRequest req) {
|
||||
String param = extractRequiredParameter(req, CommitLogFanoutAction.BUCKET_PARAM);
|
||||
Integer bucket = Ints.tryParse(param);
|
||||
if (bucket == null) {
|
||||
throw new BadRequestException("Bad bucket id");
|
||||
}
|
||||
return bucket;
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(LOWER_CHECKPOINT_TIME_PARAM)
|
||||
static DateTime provideLowerCheckpointKey(HttpServletRequest req) {
|
||||
return extractRequiredDatetimeParameter(req, LOWER_CHECKPOINT_TIME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(UPPER_CHECKPOINT_TIME_PARAM)
|
||||
static DateTime provideUpperCheckpointKey(HttpServletRequest req) {
|
||||
return extractRequiredDatetimeParameter(req, UPPER_CHECKPOINT_TIME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(BUCKET_OVERRIDE_PARAM)
|
||||
static Optional<String> provideBucketOverride(HttpServletRequest req) {
|
||||
return extractOptionalParameter(req, BUCKET_OVERRIDE_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(FROM_TIME_PARAM)
|
||||
static DateTime provideFromTime(HttpServletRequest req) {
|
||||
return extractRequiredDatetimeParameter(req, FROM_TIME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(TO_TIME_PARAM)
|
||||
static DateTime provideToTime(HttpServletRequest req) {
|
||||
return extractRequiredDatetimeParameter(req, TO_TIME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Backups
|
||||
static ListeningExecutorService provideListeningExecutorService() {
|
||||
return listeningDecorator(newFixedThreadPool(NUM_THREADS, currentRequestThreadFactory()));
|
||||
}
|
||||
|
||||
@Provides
|
||||
static ScheduledExecutorService provideScheduledExecutorService() {
|
||||
return Executors.newSingleThreadScheduledExecutor();
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
|
||||
import com.google.appengine.api.datastore.EntityTranslator;
|
||||
import com.google.common.collect.AbstractIterator;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.Iterator;
|
||||
|
||||
/** Utilities for working with backups. */
|
||||
@DeleteAfterMigration
|
||||
public class BackupUtils {
|
||||
|
||||
/** Keys for user metadata fields on commit log files in GCS. */
|
||||
public static final class GcsMetadataKeys {
|
||||
|
||||
private GcsMetadataKeys() {}
|
||||
|
||||
public static final String NUM_TRANSACTIONS = "num_transactions";
|
||||
public static final String LOWER_BOUND_CHECKPOINT = "lower_bound_checkpoint";
|
||||
public static final String UPPER_BOUND_CHECKPOINT = "upper_bound_checkpoint";
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
|
||||
* OutputStream} in delimited protocol buffer format.
|
||||
*/
|
||||
static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
|
||||
EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnlyWithoutBackup().toEntity(entity))
|
||||
.writeDelimitedTo(stream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an iterator of {@link ImmutableObject} instances deserialized from the given stream.
|
||||
*
|
||||
* <p>This parses out delimited protocol buffers for raw Datastore entities and then Ofy-loads
|
||||
* those as {@link ImmutableObject}.
|
||||
*
|
||||
* <p>The iterator reads from the stream on demand, and as such will fail if the stream is closed.
|
||||
*/
|
||||
public static Iterator<ImmutableObject> createDeserializingIterator(
|
||||
final InputStream input, boolean withAppIdOverride) {
|
||||
return new AbstractIterator<ImmutableObject>() {
|
||||
@Override
|
||||
protected ImmutableObject computeNext() {
|
||||
EntityProto proto = new EntityProto();
|
||||
if (proto.parseDelimitedFrom(input)) { // False means end of stream; other errors throw.
|
||||
if (withAppIdOverride) {
|
||||
proto = EntityImports.fixEntity(proto);
|
||||
}
|
||||
return auditedOfy().load().fromEntity(EntityTranslator.createFromPb(proto));
|
||||
}
|
||||
return endOfData();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public static ImmutableList<ImmutableObject> deserializeEntities(byte[] bytes) {
|
||||
return ImmutableList.copyOf(
|
||||
createDeserializingIterator(new ByteArrayInputStream(bytes), false));
|
||||
}
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Service;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/**
|
||||
* Action that saves commit log checkpoints to Datastore and kicks off a diff export task.
|
||||
*
|
||||
* <p>We separate computing and saving the checkpoint from exporting it because the export to GCS is
|
||||
* retryable but should not require the computation of a new checkpoint. Saving the checkpoint and
|
||||
* enqueuing the export task are done transactionally, so any checkpoint that is saved will be
|
||||
* exported to GCS very soon.
|
||||
*
|
||||
* <p>This action's supported method is GET rather than POST because it gets invoked via cron.
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/cron/commitLogCheckpoint",
|
||||
method = Action.Method.GET,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public final class CommitLogCheckpointAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private static final String QUEUE_NAME = "export-commits";
|
||||
|
||||
/**
|
||||
* The amount of time enqueueing should be delayed.
|
||||
*
|
||||
* <p>The {@link ExportCommitLogDiffAction} is enqueued in {@link CommitLogCheckpointAction},
|
||||
* which is inside a Datastore transaction that persists the checkpoint to be exported. After the
|
||||
* switch to CloudTasks API, the task may be invoked before the Datastore transaction commits.
|
||||
* When this happens, the checkpoint is not found which leads to {@link
|
||||
* com.google.common.base.VerifyException}.
|
||||
*
|
||||
* <p>In order to invoke the task after the transaction commits, a reasonable delay should be
|
||||
* added to each task. The latency of the request is mostly in the range of 4-6 seconds; Choosing
|
||||
* a value 30% greater than the upper bound should solve the issue invoking a task before the
|
||||
* transaction commits.
|
||||
*/
|
||||
static final Duration ENQUEUE_DELAY_SECONDS = Duration.standardSeconds(8);
|
||||
|
||||
@Inject CommitLogCheckpointStrategy strategy;
|
||||
@Inject CloudTasksUtils cloudTasksUtils;
|
||||
|
||||
@Inject CommitLogCheckpointAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
createCheckPointAndStartAsyncExport();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@link CommitLogCheckpoint} and initiates an asynchronous export task.
|
||||
*
|
||||
* @return the {@code CommitLogCheckpoint} to be exported
|
||||
*/
|
||||
public Optional<CommitLogCheckpoint> createCheckPointAndStartAsyncExport() {
|
||||
final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
|
||||
logger.atInfo().log(
|
||||
"Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
|
||||
boolean isCheckPointPersisted =
|
||||
ofyTm()
|
||||
.transact(
|
||||
() -> {
|
||||
DateTime lastWrittenTime =
|
||||
CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
|
||||
if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {
|
||||
logger.atInfo().log(
|
||||
"Newer checkpoint already written at time: %s", lastWrittenTime);
|
||||
return false;
|
||||
}
|
||||
auditedOfy()
|
||||
.saveIgnoringReadOnlyWithoutBackup()
|
||||
.entities(
|
||||
checkpoint,
|
||||
CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
|
||||
// Enqueue a diff task between previous and current checkpoints.
|
||||
cloudTasksUtils.enqueue(
|
||||
QUEUE_NAME,
|
||||
cloudTasksUtils.createPostTaskWithDelay(
|
||||
ExportCommitLogDiffAction.PATH,
|
||||
Service.BACKEND.toString(),
|
||||
ImmutableMultimap.of(
|
||||
LOWER_CHECKPOINT_TIME_PARAM,
|
||||
lastWrittenTime.toString(),
|
||||
UPPER_CHECKPOINT_TIME_PARAM,
|
||||
checkpoint.getCheckpointTime().toString()),
|
||||
ENQUEUE_DELAY_SECONDS));
|
||||
return true;
|
||||
});
|
||||
return isCheckPointPersisted ? Optional.of(checkpoint) : Optional.empty();
|
||||
}
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
||||
import static com.google.common.collect.Maps.transformValues;
|
||||
import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
|
||||
import static google.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.earliestOf;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.List;
|
||||
import java.util.Map.Entry;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* Implementation of the procedure for determining point-in-time consistent commit log checkpoint.
|
||||
*
|
||||
* <p>This algorithm examines the recently written commit log data and uses a dual-read approach to
|
||||
* determine a point-in-time consistent set of checkpoint times for the commit log buckets. By
|
||||
* "consistent" we mean, generally speaking, that if the Datastore were restored by replaying all
|
||||
* the commit logs up to the checkpoint times of the buckets, the result would be transactionally
|
||||
* correct; there must be no "holes" where restored state depends on non-restored state.
|
||||
*
|
||||
* <p>The consistency guarantee really has two parts, only one of which is provided by this
|
||||
* algorithm. The procedure below guarantees only that if the resulting checkpoint includes any
|
||||
* given commit log, it will also include all the commit logs that were both 1) actually written
|
||||
* before that commit log "in real life", and 2) have an earlier timestamp than that commit log.
|
||||
* (These criteria do not necessarily imply each other, due to the lack of a global shared clock.)
|
||||
* The rest of the guarantee comes from our Ofy customizations, which ensure that any transaction
|
||||
* that depends on state from a previous transaction does indeed have a later timestamp.
|
||||
*
|
||||
* <h2>Procedure description</h2>
|
||||
*
|
||||
* <pre>{@code
|
||||
* ComputeCheckpoint() -> returns a set consisting of a timestamp c(b_i) for every bucket b_i
|
||||
*
|
||||
* 1) read off the latest commit timestamp t(b_i) for every bucket b_i
|
||||
* 2) iterate over the buckets b_i a second time, and
|
||||
* a) do a consistent query for the next commit timestamp t'(b_i) where t'(b_i) > t(b_i)
|
||||
* b) if present, add this timestamp t'(b_i) to a set S
|
||||
* 3) compute a threshold time T* representing a time before all commits in S, as follows:
|
||||
* a) if S is empty, let T* = +∞ (or the "end of time")
|
||||
* b) else, let T* = T - Δ, for T = min(S) and some small Δ > 0
|
||||
* 4) return the set given by: min(t(b_i), T*) for all b_i
|
||||
* }</pre>
|
||||
*
|
||||
* <h2>Correctness proof of algorithm</h2>
|
||||
*
|
||||
* <p>{@literal As described above, the algorithm is correct as long as it can ensure the following:
|
||||
* given a commit log X written at time t(X) to bucket b_x, and another commit log Y that was
|
||||
* written "in real life" before X and for which t(Y) < t(X), then if X is included in the
|
||||
* checkpoint, so is Y; that is, t(X) <= c(b_x) implies t(Y) <= c(b_y). }
|
||||
*
|
||||
* <p>{@literal To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e.
|
||||
* every commit log included in the checkpoint must have been seen in the first pass. Hence if X was
|
||||
* included, then X must have been written by the time we started the second pass. But since Y was
|
||||
* written "in real life" prior to X, we must have seen Y by the second pass too. }
|
||||
*
|
||||
* <p>{@literal Now assume towards a contradiction that X is indeed included but Y is not, i.e. that
|
||||
* we have t(X) <= c(b_x) but t(Y) > c(b_y). If Y was seen in the first pass, i.e. t(Y) <= t(b_y),
|
||||
* then by our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y). By the definition
|
||||
* of c(b_y) it must then equal T*, so we have T* < t(Y). However, this is a contradiction since
|
||||
* t(Y) < t(X) and t(X) <= c(b_x) <= T*. If instead Y was seen in the second pass but not the first,
|
||||
* t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by definition,
|
||||
* we again reach the contradiction T* < t(Y). }
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
class CommitLogCheckpointStrategy {
|
||||
|
||||
@Inject Ofy ofy;
|
||||
@Inject Clock clock;
|
||||
@Inject CommitLogCheckpointStrategy() {}
|
||||
|
||||
/** Compute and return a new CommitLogCheckpoint for the current point in time. */
|
||||
public CommitLogCheckpoint computeCheckpoint() {
|
||||
DateTime checkpointTime = clock.nowUtc();
|
||||
ImmutableMap<Integer, DateTime> firstPassTimes = readBucketTimestamps();
|
||||
DateTime threshold = readNewCommitLogsAndFindThreshold(firstPassTimes);
|
||||
return CommitLogCheckpoint.create(
|
||||
checkpointTime,
|
||||
computeBucketCheckpointTimes(firstPassTimes, threshold));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a map from all bucket IDs to their current last written time values, fetched without
|
||||
* a transaction so with no guarantee of consistency across buckets.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
ImmutableMap<Integer, DateTime> readBucketTimestamps() {
|
||||
// Use a fresh session cache so that we get the latest data from Datastore.
|
||||
return ofy.doWithFreshSessionCache(
|
||||
() ->
|
||||
CommitLogBucket.loadAllBuckets()
|
||||
.stream()
|
||||
.collect(
|
||||
ImmutableMap.toImmutableMap(
|
||||
CommitLogBucket::getBucketNum, CommitLogBucket::getLastWrittenTime)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a threshold value defined as the latest timestamp that is before all new commit logs,
|
||||
* where "new" means having a commit time after the per-bucket timestamp in the given map.
|
||||
* When no such commit logs exist, the threshold value is set to END_OF_TIME.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
DateTime readNewCommitLogsAndFindThreshold(ImmutableMap<Integer, DateTime> bucketTimes) {
|
||||
DateTime timeBeforeAllNewCommits = END_OF_TIME;
|
||||
for (Entry<Integer, DateTime> entry : bucketTimes.entrySet()) {
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(entry.getKey());
|
||||
DateTime bucketTime = entry.getValue();
|
||||
// Add 1 to handle START_OF_TIME since 0 isn't a valid id - filter then uses >= instead of >.
|
||||
Key<CommitLogManifest> keyForFilter =
|
||||
Key.create(CommitLogManifest.create(bucketKey, bucketTime.plusMillis(1), null));
|
||||
List<Key<CommitLogManifest>> manifestKeys =
|
||||
ofy.load()
|
||||
.type(CommitLogManifest.class)
|
||||
.ancestor(bucketKey)
|
||||
.filterKey(">=", keyForFilter)
|
||||
.limit(1)
|
||||
.keys()
|
||||
.list();
|
||||
if (!manifestKeys.isEmpty()) {
|
||||
timeBeforeAllNewCommits = earliestOf(
|
||||
timeBeforeAllNewCommits,
|
||||
CommitLogManifest.extractCommitTime(getOnlyElement(manifestKeys)).minusMillis(1));
|
||||
}
|
||||
}
|
||||
return timeBeforeAllNewCommits;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the bucket checkpoint times produced by clamping the given set of bucket timestamps to
|
||||
* at most the given threshold value.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
ImmutableMap<Integer, DateTime> computeBucketCheckpointTimes(
|
||||
ImmutableMap<Integer, DateTime> firstPassTimes,
|
||||
final DateTime threshold) {
|
||||
return ImmutableMap.copyOf(
|
||||
transformValues(firstPassTimes, firstPassTime -> earliestOf(firstPassTime, threshold)));
|
||||
}
|
||||
}
|
||||
@@ -1,346 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
|
||||
import static java.lang.Boolean.FALSE;
|
||||
import static java.lang.Boolean.TRUE;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMultiset;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.CommitLogManifestInput;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/**
|
||||
* Task that garbage collects old {@link CommitLogManifest} entities.
|
||||
*
|
||||
* <p>Once commit logs have been written to GCS, we don't really need them in Datastore anymore,
|
||||
* except to reconstruct point-in-time snapshots of the database. To make that possible, {@link
|
||||
* EppResource}s have a {@link EppResource#getRevisions} method that returns the commit logs for
|
||||
* older points in time. But that functionality is not useful after a certain amount of time, e.g.
|
||||
* thirty days, so unneeded revisions are deleted (see {@link CommitLogRevisionsTranslatorFactory}).
|
||||
* This leaves commit logs in the system that are unneeded (have no revisions pointing to them). So
|
||||
* this task runs periodically to delete the "orphan" commit logs.
|
||||
*
|
||||
* <p>This action runs a mapreduce that goes over all existing {@link EppResource} and all {@link
|
||||
* CommitLogManifest} older than commitLogDatastreRetention, and erases the commit logs aren't in an
|
||||
* EppResource.
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/deleteOldCommitLogs",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
// No longer needed in SQL. Subject to future removal.
|
||||
@Deprecated
|
||||
@DeleteAfterMigration
|
||||
public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
|
||||
private static final int NUM_MAP_SHARDS = 20;
|
||||
private static final int NUM_REDUCE_SHARDS = 10;
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
|
||||
@Inject
|
||||
@Config("commitLogDatastoreRetention")
|
||||
Duration maxAge;
|
||||
|
||||
@Inject
|
||||
@Parameter(PARAM_DRY_RUN)
|
||||
boolean isDryRun;
|
||||
|
||||
@Inject
|
||||
DeleteOldCommitLogsAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
DateTime deletionThreshold = clock.nowUtc().minus(maxAge);
|
||||
logger.atInfo().log(
|
||||
"Processing asynchronous deletion of unreferenced CommitLogManifests older than %s.",
|
||||
deletionThreshold);
|
||||
|
||||
mrRunner
|
||||
.setJobName("Delete old commit logs")
|
||||
.setModuleName("backend")
|
||||
.setDefaultMapShards(NUM_MAP_SHARDS)
|
||||
.setDefaultReduceShards(NUM_REDUCE_SHARDS)
|
||||
.runMapreduce(
|
||||
new DeleteOldCommitLogsMapper(deletionThreshold),
|
||||
new DeleteOldCommitLogsReducer(deletionThreshold, isDryRun),
|
||||
ImmutableList.of(
|
||||
new CommitLogManifestInput(deletionThreshold),
|
||||
EppResourceInputs.createKeyInput(EppResource.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
}
|
||||
|
||||
/**
|
||||
* A mapper that iterates over all {@link EppResource} and {CommitLogManifest} entities.
|
||||
*
|
||||
* <p>It emits the target key and {@code false} for all revisions of each EppResources (meaning
|
||||
* "don't delete this"), and {@code true} for all CommitLogRevisions (meaning "delete this").
|
||||
*
|
||||
* <p>The reducer will then delete all CommitLogRevisions that only have {@code true}.
|
||||
*/
|
||||
private static class DeleteOldCommitLogsMapper
|
||||
extends Mapper<Key<?>, Key<CommitLogManifest>, Boolean> {
|
||||
|
||||
private static final long serialVersionUID = 8008689353479902948L;
|
||||
|
||||
private static final String KIND_MANIFEST = Key.getKind(CommitLogManifest.class);
|
||||
|
||||
private final DateTime threshold;
|
||||
|
||||
DeleteOldCommitLogsMapper(DateTime threshold) {
|
||||
this.threshold = threshold;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void map(final Key<?> key) {
|
||||
// key is either a Key<CommitLogManifest> or a Key<? extends EppResource>.
|
||||
//
|
||||
// If it's a CommitLogManifest we just emit it as is (no need to load it).
|
||||
if (key.getKind().equals(KIND_MANIFEST)) {
|
||||
getContext().incrementCounter("old commit log manifests found");
|
||||
// safe because we checked getKind
|
||||
@SuppressWarnings("unchecked")
|
||||
Key<CommitLogManifest> manifestKey = (Key<CommitLogManifest>) key;
|
||||
emit(manifestKey, true);
|
||||
return;
|
||||
}
|
||||
|
||||
// If it isn't a Key<CommitLogManifest> then it should be an EppResource, which we need to
|
||||
// load to emit the revisions.
|
||||
//
|
||||
Object object = auditedOfy().load().key(key).now();
|
||||
checkNotNull(object, "Received a key to a missing object. key: %s", key);
|
||||
checkState(
|
||||
object instanceof EppResource,
|
||||
"Received a key to an object that isn't EppResource nor CommitLogManifest."
|
||||
+ " Key: %s object type: %s",
|
||||
key,
|
||||
object.getClass().getName());
|
||||
|
||||
getContext().incrementCounter("EPP resources found");
|
||||
EppResource eppResource = (EppResource) object;
|
||||
if (eppResource.getCreationTime().isAfter(threshold)) {
|
||||
getContext().incrementCounter("EPP resources newer than threshold");
|
||||
}
|
||||
for (Key<CommitLogManifest> manifestKey : eppResource.getRevisions().values()) {
|
||||
emit(manifestKey, false);
|
||||
}
|
||||
getContext()
|
||||
.incrementCounter("EPP resource revisions found", eppResource.getRevisions().size());
|
||||
checkAndLogRevisionCoverageError(eppResource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if given eppResource has the required revisions.
|
||||
*
|
||||
* <p>Revisions are used to recreate the state of the resource at a given day in the past
|
||||
* "commitLogDatastoreRenention". To do that, we need at least one revision that's older than
|
||||
* this duration (is dated before "threshold"), or at least one revision within a day of the
|
||||
* resource's creation if it was created after the threshold.
|
||||
*
|
||||
* <p>Here we check that the given eppResource has the revisions it needs.
|
||||
*
|
||||
* <p>It's just a sanity check - since we're relying on the revisions to be correct for the
|
||||
* deletion to work. We want to alert any problems we find in the revisions.
|
||||
*
|
||||
* <p>This really checks {@link CommitLogRevisionsTranslatorFactory#transformBeforeSave}.
|
||||
* There's nothing we can do at this point to prevent the damage - we only report on it.
|
||||
*/
|
||||
private void checkAndLogRevisionCoverageError(EppResource eppResource) {
|
||||
// First - check if there even are revisions
|
||||
if (eppResource.getRevisions().isEmpty()) {
|
||||
getContext().incrementCounter("EPP resources missing all revisions (SEE LOGS)");
|
||||
logger.atSevere().log("EPP resource missing all revisions: %s", Key.create(eppResource));
|
||||
return;
|
||||
}
|
||||
// Next, check if there's a revision that's older than "CommitLogDatastoreRetention". There
|
||||
// should have been at least one at the time this resource was saved.
|
||||
//
|
||||
// Alternatively, if the resource is newer than the threshold - there should be at least one
|
||||
// revision within a day of the creation time.
|
||||
DateTime oldestRevisionDate = eppResource.getRevisions().firstKey();
|
||||
if (oldestRevisionDate.isBefore(threshold)
|
||||
|| oldestRevisionDate.isBefore(eppResource.getCreationTime().plusDays(1))) {
|
||||
// We're OK!
|
||||
return;
|
||||
}
|
||||
// The oldest revision date is newer than the threshold! This shouldn't happen.
|
||||
getContext().incrementCounter("EPP resources missing pre-threshold revision (SEE LOGS)");
|
||||
logger.atSevere().log(
|
||||
"EPP resource missing old enough revision: "
|
||||
+ "%s (created on %s) has %d revisions between %s and %s, while threshold is %s.",
|
||||
Key.create(eppResource),
|
||||
eppResource.getCreationTime(),
|
||||
eppResource.getRevisions().size(),
|
||||
eppResource.getRevisions().firstKey(),
|
||||
eppResource.getRevisions().lastKey(),
|
||||
threshold);
|
||||
// We want to see how bad it is though: if the difference is less than a day then this might
|
||||
// still be OK (we only need logs for the end of the day). But if it's more than a day, then
|
||||
// we are 100% sure we can't recreate all the history we need from the revisions.
|
||||
Duration interval = new Duration(threshold, oldestRevisionDate);
|
||||
if (interval.isLongerThan(Duration.standardDays(1))) {
|
||||
getContext()
|
||||
.incrementCounter("EPP resources missing pre-(threshold+1d) revision (SEE LOGS)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reducer that deletes unreferenced {@link CommitLogManifest} + child {@link CommitLogMutation}.
|
||||
*
|
||||
* <p>It receives the manifestKey to possibly delete, and a list of boolean 'verdicts' from
|
||||
* various sources (the "old manifests" source and the "still referenced" source) on whether it's
|
||||
* OK to delete this manifestKey. If even one source returns "false" (meaning "it's not OK to
|
||||
* delete this manifest") then it won't be deleted.
|
||||
*/
|
||||
static class DeleteOldCommitLogsReducer extends Reducer<Key<CommitLogManifest>, Boolean, Void> {
|
||||
|
||||
private static final long serialVersionUID = -4918760187627937268L;
|
||||
|
||||
private final DateTime deletionThreshold;
|
||||
private final boolean isDryRun;
|
||||
|
||||
@AutoValue
|
||||
abstract static class DeletionResult {
|
||||
enum Status {
|
||||
ALREADY_DELETED,
|
||||
AFTER_THRESHOLD,
|
||||
SUCCESS
|
||||
}
|
||||
|
||||
public abstract Status status();
|
||||
|
||||
public abstract int numDeleted();
|
||||
|
||||
static DeletionResult create(Status status, int numDeleted) {
|
||||
return new AutoValue_DeleteOldCommitLogsAction_DeleteOldCommitLogsReducer_DeletionResult(
|
||||
status, numDeleted);
|
||||
}
|
||||
}
|
||||
|
||||
DeleteOldCommitLogsReducer(DateTime deletionThreshold, boolean isDryRun) {
|
||||
this.deletionThreshold = deletionThreshold;
|
||||
this.isDryRun = isDryRun;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reduce(
|
||||
final Key<CommitLogManifest> manifestKey, ReducerInput<Boolean> canDeleteVerdicts) {
|
||||
ImmutableMultiset<Boolean> canDeleteMultiset = ImmutableMultiset.copyOf(canDeleteVerdicts);
|
||||
if (canDeleteMultiset.count(TRUE) > 1) {
|
||||
getContext().incrementCounter("commit log manifests incorrectly mapped multiple times");
|
||||
}
|
||||
if (canDeleteMultiset.count(FALSE) > 1) {
|
||||
getContext().incrementCounter("commit log manifests referenced multiple times");
|
||||
}
|
||||
if (canDeleteMultiset.contains(FALSE)) {
|
||||
getContext()
|
||||
.incrementCounter(
|
||||
canDeleteMultiset.contains(TRUE)
|
||||
? "old commit log manifests still referenced"
|
||||
: "new (or nonexistent) commit log manifests referenced");
|
||||
getContext()
|
||||
.incrementCounter("EPP resource revisions handled", canDeleteMultiset.count(FALSE));
|
||||
return;
|
||||
}
|
||||
|
||||
DeletionResult deletionResult =
|
||||
ofyTm()
|
||||
.transactNew(
|
||||
() -> {
|
||||
CommitLogManifest manifest = auditedOfy().load().key(manifestKey).now();
|
||||
// It is possible that the same manifestKey was run twice, if a shard had to be
|
||||
// restarted or some weird failure. If this happens, we want to exit
|
||||
// immediately. Note that this can never happen in dryRun.
|
||||
if (manifest == null) {
|
||||
return DeletionResult.create(DeletionResult.Status.ALREADY_DELETED, 0);
|
||||
}
|
||||
// Doing a sanity check on the date. This is the only place we use the
|
||||
// CommitLogManifest, so maybe removing this test will improve performance.
|
||||
// However, unless it's proven that the performance boost is significant (and
|
||||
// we've tested this enough to be sure it never happens)- the safety of "let's
|
||||
// not delete stuff we need from prod" is more important.
|
||||
if (manifest.getCommitTime().isAfter(deletionThreshold)) {
|
||||
return DeletionResult.create(DeletionResult.Status.AFTER_THRESHOLD, 0);
|
||||
}
|
||||
Iterable<Key<CommitLogMutation>> commitLogMutationKeys =
|
||||
auditedOfy()
|
||||
.load()
|
||||
.type(CommitLogMutation.class)
|
||||
.ancestor(manifestKey)
|
||||
.keys()
|
||||
.iterable();
|
||||
ImmutableList<Key<?>> keysToDelete =
|
||||
ImmutableList.<Key<?>>builder()
|
||||
.addAll(commitLogMutationKeys)
|
||||
.add(manifestKey)
|
||||
.build();
|
||||
// Normally in a dry run we would log the entities that would be deleted, but
|
||||
// those can number in the millions so we skip the logging.
|
||||
if (!isDryRun) {
|
||||
auditedOfy().deleteWithoutBackup().keys(keysToDelete);
|
||||
}
|
||||
return DeletionResult.create(
|
||||
DeletionResult.Status.SUCCESS, keysToDelete.size());
|
||||
});
|
||||
|
||||
switch (deletionResult.status()) {
|
||||
case SUCCESS:
|
||||
getContext().incrementCounter("old commit log manifests deleted");
|
||||
getContext().incrementCounter("total entities deleted", deletionResult.numDeleted());
|
||||
break;
|
||||
case ALREADY_DELETED:
|
||||
getContext().incrementCounter("attempts to delete an already deleted manifest");
|
||||
break;
|
||||
case AFTER_THRESHOLD:
|
||||
logger.atSevere().log(
|
||||
"Won't delete CommitLogManifest %s that is too recent.", manifestKey);
|
||||
getContext().incrementCounter("manifests incorrectly assigned for deletion (SEE LOGS)");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import com.google.apphosting.api.ApiProxy;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.Path;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.util.Objects;
|
||||
|
||||
/** Utilities for handling imported Datastore entities. */
|
||||
@DeleteAfterMigration
|
||||
public class EntityImports {
|
||||
|
||||
/**
|
||||
* Transitively sets the {@code appId} of all keys in a foreign entity to that of the current
|
||||
* system.
|
||||
*/
|
||||
public static EntityProto fixEntity(EntityProto entityProto) {
|
||||
String currentAappId = ApiProxy.getCurrentEnvironment().getAppId();
|
||||
if (Objects.equals(currentAappId, entityProto.getKey().getApp())) {
|
||||
return entityProto;
|
||||
}
|
||||
return fixEntity(entityProto, currentAappId);
|
||||
}
|
||||
|
||||
private static EntityProto fixEntity(EntityProto entityProto, String appId) {
|
||||
if (entityProto.hasKey()) {
|
||||
fixKey(entityProto, appId);
|
||||
}
|
||||
|
||||
for (OnestoreEntity.Property property : entityProto.mutablePropertys()) {
|
||||
fixProperty(property, appId);
|
||||
}
|
||||
|
||||
for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
|
||||
fixProperty(property, appId);
|
||||
}
|
||||
|
||||
// CommitLogMutation embeds an entity as bytes, which needs additional fixes.
|
||||
if (isCommitLogMutation(entityProto)) {
|
||||
fixMutationEntityProtoBytes(entityProto, appId);
|
||||
}
|
||||
return entityProto;
|
||||
}
|
||||
|
||||
private static boolean isCommitLogMutation(EntityProto entityProto) {
|
||||
if (!entityProto.hasKey()) {
|
||||
return false;
|
||||
}
|
||||
Path path = entityProto.getKey().getPath();
|
||||
if (path.elementSize() == 0) {
|
||||
return false;
|
||||
}
|
||||
return Objects.equals(path.getElement(path.elementSize() - 1).getType(), "CommitLogMutation");
|
||||
}
|
||||
|
||||
private static void fixMutationEntityProtoBytes(EntityProto entityProto, String appId) {
|
||||
for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
|
||||
if (Objects.equals(property.getName(), "entityProtoBytes")) {
|
||||
OnestoreEntity.PropertyValue value = property.getValue();
|
||||
EntityProto fixedProto =
|
||||
fixEntity(bytesToEntityProto(value.getStringValueAsBytes()), appId);
|
||||
value.setStringValueAsBytes(fixedProto.toByteArray());
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void fixKey(EntityProto entityProto, String appId) {
|
||||
entityProto.getMutableKey().setApp(appId);
|
||||
}
|
||||
|
||||
private static void fixKey(ReferenceValue referenceValue, String appId) {
|
||||
referenceValue.setApp(appId);
|
||||
}
|
||||
|
||||
private static void fixProperty(OnestoreEntity.Property property, String appId) {
|
||||
OnestoreEntity.PropertyValue value = property.getMutableValue();
|
||||
if (value.hasReferenceValue()) {
|
||||
fixKey(value.getMutableReferenceValue(), appId);
|
||||
return;
|
||||
}
|
||||
if (property.getMeaningEnum().equals(Meaning.ENTITY_PROTO)) {
|
||||
EntityProto embeddedProto = bytesToEntityProto(value.getStringValueAsBytes());
|
||||
fixEntity(embeddedProto, appId);
|
||||
value.setStringValueAsBytes(embeddedProto.toByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
private static EntityProto bytesToEntityProto(byte[] bytes) {
|
||||
EntityProto entityProto = new EntityProto();
|
||||
boolean isParsed = entityProto.parseFrom(bytes);
|
||||
if (!isParsed) {
|
||||
throw new IllegalStateException("Failed to parse raw bytes as EntityProto.");
|
||||
}
|
||||
return entityProto;
|
||||
}
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Verify.verifyNotNull;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterables.concat;
|
||||
import static com.google.common.collect.Lists.partition;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.NUM_TRANSACTIONS;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.UPPER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.BackupUtils.serializeEntity;
|
||||
import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static java.util.Comparator.comparingLong;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.auth.Auth;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Action that exports the diff between two commit log checkpoints to GCS. */
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = ExportCommitLogDiffAction.PATH,
|
||||
method = Action.Method.POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public final class ExportCommitLogDiffAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
static final String PATH = "/_dr/task/exportCommitLogDiff";
|
||||
static final String UPPER_CHECKPOINT_TIME_PARAM = "upperCheckpointTime";
|
||||
static final String LOWER_CHECKPOINT_TIME_PARAM = "lowerCheckpointTime";
|
||||
|
||||
public static final String DIFF_FILE_PREFIX = "commit_diff_until_";
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
@Inject @Config("commitLogDiffExportBatchSize") int batchSize;
|
||||
@Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime;
|
||||
@Inject @Parameter(UPPER_CHECKPOINT_TIME_PARAM) DateTime upperCheckpointTime;
|
||||
@Inject ExportCommitLogDiffAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
logger.atInfo().log(
|
||||
"Exporting commit log diffs between %s and %s.", lowerCheckpointTime, upperCheckpointTime);
|
||||
checkArgument(isAtOrAfter(lowerCheckpointTime, START_OF_TIME));
|
||||
checkArgument(lowerCheckpointTime.isBefore(upperCheckpointTime));
|
||||
// Load the boundary checkpoints - lower is exclusive and may not exist (on the first export,
|
||||
// when lowerCheckpointTime is START_OF_TIME), whereas the upper is inclusive and must exist.
|
||||
CommitLogCheckpoint lowerCheckpoint =
|
||||
lowerCheckpointTime.isAfter(START_OF_TIME)
|
||||
? verifyNotNull(
|
||||
auditedOfy().load().key(CommitLogCheckpoint.createKey(lowerCheckpointTime)).now())
|
||||
: null;
|
||||
CommitLogCheckpoint upperCheckpoint =
|
||||
verifyNotNull(
|
||||
auditedOfy().load().key(CommitLogCheckpoint.createKey(upperCheckpointTime)).now());
|
||||
|
||||
// Load the keys of all the manifests to include in this diff.
|
||||
List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
|
||||
logger.atInfo().log("Found %d manifests to export.", sortedKeys.size());
|
||||
// Open an output channel to GCS, wrapped in a stream for convenience.
|
||||
try (OutputStream gcsStream =
|
||||
gcsUtils.openOutputStream(
|
||||
BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
ImmutableMap.of(
|
||||
LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(),
|
||||
UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(),
|
||||
NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) {
|
||||
// Export the upper checkpoint itself.
|
||||
serializeEntity(upperCheckpoint, gcsStream);
|
||||
// If there are no manifests to export, stop early, now that we've written out the file with
|
||||
// the checkpoint itself (which is needed for restores, even if it's empty).
|
||||
if (sortedKeys.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// Export to GCS in chunks, one per fixed batch of commit logs. While processing one batch,
|
||||
// asynchronously load the entities for the next one.
|
||||
List<List<Key<CommitLogManifest>>> keyChunks = partition(sortedKeys, batchSize);
|
||||
// Objectify's map return type is asynchronous. Calling .values() will block until it loads.
|
||||
Map<?, CommitLogManifest> nextChunkToExport = auditedOfy().load().keys(keyChunks.get(0));
|
||||
for (int i = 0; i < keyChunks.size(); i++) {
|
||||
// Force the async load to finish.
|
||||
Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
|
||||
logger.atInfo().log("Loaded %d manifests.", chunkValues.size());
|
||||
// Since there is no hard bound on how much data this might be, take care not to let the
|
||||
// Objectify session cache fill up and potentially run out of memory. This is the only safe
|
||||
// point to do this since at this point there is no async load in progress.
|
||||
auditedOfy().clearSessionCache();
|
||||
// Kick off the next async load, which can happen in parallel to the current GCS export.
|
||||
if (i + 1 < keyChunks.size()) {
|
||||
nextChunkToExport = auditedOfy().load().keys(keyChunks.get(i + 1));
|
||||
}
|
||||
exportChunk(gcsStream, chunkValues);
|
||||
logger.atInfo().log("Exported %d manifests.", chunkValues.size());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
logger.atInfo().log("Exported %d total manifests.", sortedKeys.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads all the diff keys, sorted in a transaction-consistent chronological order.
|
||||
*
|
||||
* @param lowerCheckpoint exclusive lower bound on keys in this diff, or null if no lower bound
|
||||
* @param upperCheckpoint inclusive upper bound on keys in this diff
|
||||
*/
|
||||
private ImmutableList<Key<CommitLogManifest>> loadAllDiffKeys(
|
||||
@Nullable final CommitLogCheckpoint lowerCheckpoint,
|
||||
final CommitLogCheckpoint upperCheckpoint) {
|
||||
// Fetch the keys (no data) between these checkpoints, and sort by timestamp. This ordering is
|
||||
// transaction-consistent by virtue of our checkpoint strategy and our customized Ofy; see
|
||||
// CommitLogCheckpointStrategy for the proof. We break ties by sorting on bucket ID to ensure
|
||||
// a deterministic order.
|
||||
return upperCheckpoint
|
||||
.getBucketTimestamps()
|
||||
.keySet()
|
||||
.stream()
|
||||
.flatMap(
|
||||
bucketNum ->
|
||||
Streams.stream(loadDiffKeysFromBucket(lowerCheckpoint, upperCheckpoint, bucketNum)))
|
||||
.sorted(
|
||||
comparingLong(Key<CommitLogManifest>::getId)
|
||||
.thenComparingLong(a -> a.getParent().getId()))
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the diff keys for one bucket.
|
||||
*
|
||||
* @param lowerCheckpoint exclusive lower bound on keys in this diff, or null if no lower bound
|
||||
* @param upperCheckpoint inclusive upper bound on keys in this diff
|
||||
* @param bucketNum the bucket to load diff keys from
|
||||
*/
|
||||
private Iterable<Key<CommitLogManifest>> loadDiffKeysFromBucket(
|
||||
@Nullable CommitLogCheckpoint lowerCheckpoint,
|
||||
CommitLogCheckpoint upperCheckpoint,
|
||||
int bucketNum) {
|
||||
// If no lower checkpoint exists, or if it exists but had no timestamp for this bucket number
|
||||
// (because the bucket count was increased between these checkpoints), then use START_OF_TIME
|
||||
// as the effective exclusive lower bound.
|
||||
DateTime lowerCheckpointBucketTime =
|
||||
firstNonNull(
|
||||
(lowerCheckpoint == null) ? null : lowerCheckpoint.getBucketTimestamps().get(bucketNum),
|
||||
START_OF_TIME);
|
||||
// Since START_OF_TIME=0 is not a valid id in a key, add 1 to both bounds. Then instead of
|
||||
// loading lowerBound < x <= upperBound, we can load lowerBound <= x < upperBound.
|
||||
DateTime lowerBound = lowerCheckpointBucketTime.plusMillis(1);
|
||||
DateTime upperBound = upperCheckpoint.getBucketTimestamps().get(bucketNum).plusMillis(1);
|
||||
// If the lower and upper bounds are equal, there can't be any results, so skip the query.
|
||||
if (lowerBound.equals(upperBound)) {
|
||||
return ImmutableSet.of();
|
||||
}
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(bucketNum);
|
||||
return auditedOfy()
|
||||
.load()
|
||||
.type(CommitLogManifest.class)
|
||||
.ancestor(bucketKey)
|
||||
.filterKey(">=", CommitLogManifest.createKey(bucketKey, lowerBound))
|
||||
.filterKey("<", CommitLogManifest.createKey(bucketKey, upperBound))
|
||||
.keys();
|
||||
}
|
||||
|
||||
/** Writes a chunks-worth of manifests and associated mutations to GCS. */
|
||||
private void exportChunk(OutputStream gcsStream, Collection<CommitLogManifest> chunk)
|
||||
throws IOException {
|
||||
// Kickoff async loads for all the manifests in the chunk.
|
||||
ImmutableList.Builder<Iterable<? extends ImmutableObject>> entities =
|
||||
new ImmutableList.Builder<>();
|
||||
for (CommitLogManifest manifest : chunk) {
|
||||
entities.add(ImmutableList.of(manifest));
|
||||
entities.add(auditedOfy().load().type(CommitLogMutation.class).ancestor(manifest));
|
||||
}
|
||||
for (ImmutableObject entity : concat(entities.build())) {
|
||||
serializeEntity(entity, gcsStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,246 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static google.registry.util.DateTimeUtils.latestOf;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.common.util.concurrent.ListeningExecutorService;
|
||||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import google.registry.backup.BackupModule.Backups;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Provider;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Utility class to list commit logs diff files stored on GCS. */
|
||||
@DeleteAfterMigration
|
||||
class GcsDiffFileLister {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
/** Timeout for retrieving per-file information from GCS. */
|
||||
private static final Duration FILE_INFO_TIMEOUT_DURATION = Duration.ofMinutes(1);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Backups Provider<ListeningExecutorService> executorProvider;
|
||||
@Inject ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
@Inject
|
||||
GcsDiffFileLister() {}
|
||||
|
||||
/**
|
||||
* Traverses the sequence of diff files backwards from checkpointTime and inserts the file
|
||||
* metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
|
||||
* more files are missing.
|
||||
*
|
||||
* @throws UncheckedExecutionException wrapping a {@link java.util.concurrent.TimeoutException} if
|
||||
* the GCS call fails to finish within one minute, or wrapping any other exception if
|
||||
* something else goes wrong.
|
||||
*/
|
||||
private boolean constructDiffSequence(
|
||||
String gcsBucket,
|
||||
Map<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo,
|
||||
DateTime fromTime,
|
||||
DateTime lastTime,
|
||||
TreeMap<DateTime, BlobInfo> sequence) {
|
||||
DateTime checkpointTime = lastTime;
|
||||
while (isBeforeOrAt(fromTime, checkpointTime)) {
|
||||
BlobInfo blobInfo;
|
||||
if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) {
|
||||
blobInfo =
|
||||
Futures.getUnchecked(
|
||||
Futures.withTimeout(
|
||||
upperBoundTimesToBlobInfo.get(checkpointTime),
|
||||
FILE_INFO_TIMEOUT_DURATION,
|
||||
scheduledExecutorService));
|
||||
} else {
|
||||
String filename = DIFF_FILE_PREFIX + checkpointTime;
|
||||
logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
|
||||
blobInfo = getBlobInfo(gcsBucket, filename);
|
||||
|
||||
// If we hit a gap, quit.
|
||||
if (blobInfo == null) {
|
||||
logger.atWarning().log(
|
||||
"Gap discovered in sequence terminating at %s, missing file: %s",
|
||||
sequence.lastKey(), filename);
|
||||
logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sequence.put(checkpointTime, blobInfo);
|
||||
checkpointTime = getLowerBoundTime(blobInfo);
|
||||
}
|
||||
logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
|
||||
return true;
|
||||
}
|
||||
|
||||
ImmutableList<BlobInfo> listDiffFiles(
|
||||
String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) {
|
||||
logger.atInfo().log("Requested restore from time: %s", fromTime);
|
||||
if (toTime != null) {
|
||||
logger.atInfo().log(" Until time: %s", toTime);
|
||||
}
|
||||
// List all of the diff files on GCS and build a map from each file's upper checkpoint time
|
||||
// (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
|
||||
// an upper checkpoint time > fromTime.
|
||||
TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
|
||||
String commitLogDiffPrefix = getCommitLogDiffPrefix(fromTime, toTime);
|
||||
ImmutableList<String> filenames;
|
||||
try {
|
||||
filenames =
|
||||
gcsUtils.listFolderObjects(gcsBucket, commitLogDiffPrefix).stream()
|
||||
.map(s -> commitLogDiffPrefix + s)
|
||||
.collect(toImmutableList());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
DateTime lastUpperBoundTime = START_OF_TIME;
|
||||
|
||||
TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
|
||||
ListeningExecutorService executor = executorProvider.get();
|
||||
try {
|
||||
for (String filename : filenames) {
|
||||
String strippedFilename = filename.replaceFirst(DIFF_FILE_PREFIX, "");
|
||||
DateTime upperBoundTime = DateTime.parse(strippedFilename);
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToBlobInfo.put(
|
||||
upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
if (upperBoundTimesToBlobInfo.isEmpty()) {
|
||||
logger.atInfo().log("No files found.");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
// Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e.
|
||||
// the last file that we found) and finding its previous file until we either run out of files
|
||||
// or get to one that precedes "fromTime".
|
||||
//
|
||||
// GCS file listing is eventually consistent, so it's possible that we are missing a file. The
|
||||
// metadata of a file is sufficient to identify the preceding file, so if we start from the
|
||||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet =
|
||||
!constructDiffSequence(
|
||||
gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
// Verify that all of the elements in the original set are represented in the sequence. If we
|
||||
// find anything that's not represented, construct a sequence for it.
|
||||
boolean checkForMoreExtraDiffs = true; // Always loop at least once.
|
||||
while (checkForMoreExtraDiffs) {
|
||||
checkForMoreExtraDiffs = false;
|
||||
for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) {
|
||||
if (!isInRange(key, fromTime, toTime)) {
|
||||
break;
|
||||
}
|
||||
if (!sequence.containsKey(key)) {
|
||||
// Recalculate the sequence for purely informational purposes.
|
||||
logger.atWarning().log(
|
||||
"Fork found in commit log history. The following sequence "
|
||||
+ "is disconnected from the sequence of the final commit:");
|
||||
constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
|
||||
checkForMoreExtraDiffs = true;
|
||||
inconsistentFileSet = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checkState(
|
||||
!inconsistentFileSet,
|
||||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
} finally {
|
||||
executor.shutdown();
|
||||
}
|
||||
|
||||
logger.atInfo().log(
|
||||
"Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
|
||||
logger.atInfo().log("Found %d files to restore.", sequence.size());
|
||||
return ImmutableList.copyOf(sequence.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if 'time' is in range of 'start' and 'end'.
|
||||
*
|
||||
* <p>If 'end' is null, returns true if 'time' is after 'start'.
|
||||
*/
|
||||
private boolean isInRange(DateTime time, DateTime start, @Nullable DateTime end) {
|
||||
return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end));
|
||||
}
|
||||
|
||||
private DateTime getLowerBoundTime(BlobInfo blobInfo) {
|
||||
return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
}
|
||||
|
||||
private BlobInfo getBlobInfo(String gcsBucket, String filename) {
|
||||
return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a prefix guaranteed to cover all commit log diff files in the given range.
|
||||
*
|
||||
* <p>The listObjects call can be fairly slow if we search over many thousands or tens of
|
||||
* thousands of files, so we restrict the search space. The commit logs have a file format of
|
||||
* "commit_diff_until_2021-05-11T06:48:00.070Z" so we can often filter down as far as the hour.
|
||||
*
|
||||
* <p>Here, we get the longest prefix possible based on which fields (year, month, day, hour) the
|
||||
* times in question have in common.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static String getCommitLogDiffPrefix(DateTime from, @Nullable DateTime to) {
|
||||
StringBuilder result = new StringBuilder(DIFF_FILE_PREFIX);
|
||||
if (to == null || from.getYear() != to.getYear()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(from.getYear()).append('-');
|
||||
if (from.getMonthOfYear() != to.getMonthOfYear()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02d-", from.getMonthOfYear()));
|
||||
if (from.getDayOfMonth() != to.getDayOfMonth()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02dT", from.getDayOfMonth()));
|
||||
if (from.getHourOfDay() != to.getHourOfDay()) {
|
||||
return result.toString();
|
||||
}
|
||||
result.append(String.format("%02d:", from.getHourOfDay()));
|
||||
return result.toString();
|
||||
}
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterators.peekingIterator;
|
||||
import static google.registry.backup.BackupUtils.createDeserializingIterator;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
|
||||
import com.google.appengine.api.datastore.DatastoreService;
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.EntityTranslator;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Result;
|
||||
import com.googlecode.objectify.util.ResultNow;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Retrier;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/** Restore Registry 2 commit logs from GCS to Datastore. */
|
||||
@Action(
|
||||
service = Action.Service.TOOLS,
|
||||
path = RestoreCommitLogsAction.PATH,
|
||||
method = Action.Method.POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class RestoreCommitLogsAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
public static final String PATH = "/_dr/task/restoreCommitLogs";
|
||||
static final String DRY_RUN_PARAM = "dryRun";
|
||||
static final String FROM_TIME_PARAM = "fromTime";
|
||||
static final String TO_TIME_PARAM = "toTime";
|
||||
static final String BUCKET_OVERRIDE_PARAM = "gcsBucket";
|
||||
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Parameter(DRY_RUN_PARAM) boolean dryRun;
|
||||
@Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime;
|
||||
@Inject @Parameter(TO_TIME_PARAM) DateTime toTime;
|
||||
|
||||
@Inject
|
||||
@Parameter(BUCKET_OVERRIDE_PARAM)
|
||||
Optional<String> gcsBucketOverride;
|
||||
|
||||
@Inject DatastoreService datastoreService;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
|
||||
@Inject
|
||||
@Config("commitLogGcsBucket")
|
||||
String defaultGcsBucket;
|
||||
|
||||
@Inject Retrier retrier;
|
||||
@Inject RestoreCommitLogsAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
checkArgument(
|
||||
!FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
|
||||
"DO NOT RUN IN PRODUCTION OR SANDBOX.");
|
||||
if (dryRun) {
|
||||
logger.atInfo().log("Running in dry-run mode.");
|
||||
}
|
||||
String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
|
||||
logger.atInfo().log("Restoring from %s.", gcsBucket);
|
||||
List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
|
||||
if (diffFiles.isEmpty()) {
|
||||
logger.atInfo().log("Nothing to restore.");
|
||||
return;
|
||||
}
|
||||
Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
|
||||
CommitLogCheckpoint lastCheckpoint = null;
|
||||
for (BlobInfo metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getName());
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
PeekingIterator<ImmutableObject> commitLogs =
|
||||
peekingIterator(createDeserializingIterator(input, true));
|
||||
lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
|
||||
saveOfy(ImmutableList.of(lastCheckpoint)); // Save the checkpoint itself.
|
||||
while (commitLogs.hasNext()) {
|
||||
CommitLogManifest manifest = restoreOneTransaction(commitLogs);
|
||||
bucketTimestamps.put(manifest.getBucketId(), manifest.getCommitTime());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
// Restore the CommitLogCheckpointRoot and CommitLogBuckets.
|
||||
saveOfy(
|
||||
Streams.concat(
|
||||
bucketTimestamps
|
||||
.entrySet()
|
||||
.stream()
|
||||
.map(
|
||||
entry ->
|
||||
new CommitLogBucket.Builder()
|
||||
.setBucketNum(entry.getKey())
|
||||
.setLastWrittenTime(entry.getValue())
|
||||
.build()),
|
||||
Stream.of(CommitLogCheckpointRoot.create(lastCheckpoint.getCheckpointTime())))
|
||||
.collect(toImmutableList()));
|
||||
logger.atInfo().log("Restore complete.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore the contents of one transaction to Datastore.
|
||||
*
|
||||
* <p>The objects to delete are listed in the {@link CommitLogManifest}, which will be the first
|
||||
* object in the iterable. The objects to save follow, each as a {@link CommitLogMutation}. We
|
||||
* restore by deleting the deletes and recreating the saves from their proto form. We also save
|
||||
* the commit logs themselves back to Datastore, so that the commit log system itself is
|
||||
* transparently restored alongside the data.
|
||||
*
|
||||
* @return the manifest, for use in restoring the {@link CommitLogBucket}.
|
||||
*/
|
||||
private CommitLogManifest restoreOneTransaction(PeekingIterator<ImmutableObject> commitLogs) {
|
||||
final CommitLogManifest manifest = (CommitLogManifest) commitLogs.next();
|
||||
Result<?> deleteResult = deleteAsync(manifest.getDeletions());
|
||||
List<Entity> entitiesToSave = Lists.newArrayList(auditedOfy().save().toEntity(manifest));
|
||||
while (commitLogs.hasNext() && commitLogs.peek() instanceof CommitLogMutation) {
|
||||
CommitLogMutation mutation = (CommitLogMutation) commitLogs.next();
|
||||
entitiesToSave.add(auditedOfy().save().toEntity(mutation));
|
||||
entitiesToSave.add(EntityTranslator.createFromPbBytes(mutation.getEntityProtoBytes()));
|
||||
}
|
||||
saveRaw(entitiesToSave);
|
||||
try {
|
||||
deleteResult.now();
|
||||
} catch (Exception e) {
|
||||
retrier.callWithRetry(
|
||||
() -> deleteAsync(manifest.getDeletions()).now(), RuntimeException.class);
|
||||
}
|
||||
return manifest;
|
||||
}
|
||||
|
||||
private void saveRaw(List<Entity> entitiesToSave) {
|
||||
if (dryRun) {
|
||||
logger.atInfo().log("Would have saved entities: %s", entitiesToSave);
|
||||
return;
|
||||
}
|
||||
retrier.callWithRetry(() -> datastoreService.put(entitiesToSave), RuntimeException.class);
|
||||
}
|
||||
|
||||
private void saveOfy(Iterable<? extends ImmutableObject> objectsToSave) {
|
||||
if (dryRun) {
|
||||
logger.atInfo().log("Would have saved entities: %s", objectsToSave);
|
||||
return;
|
||||
}
|
||||
retrier.callWithRetry(
|
||||
() -> auditedOfy().saveWithoutBackup().entities(objectsToSave).now(),
|
||||
RuntimeException.class);
|
||||
}
|
||||
|
||||
private Result<?> deleteAsync(Set<Key<?>> keysToDelete) {
|
||||
if (dryRun) {
|
||||
logger.atInfo().log("Would have deleted entities: %s", keysToDelete);
|
||||
}
|
||||
return dryRun || keysToDelete.isEmpty()
|
||||
? new ResultNow<Void>(null)
|
||||
: auditedOfy().deleteWithoutBackup().keys(keysToDelete);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.backup;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.EntityTranslator;
|
||||
import com.google.appengine.api.datastore.Key;
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.auto.value.extension.memoized.Memoized;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import java.io.Serializable;
|
||||
import java.util.Arrays;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Stream;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
/**
|
||||
* A Datastore {@link Entity Entity's} serialized state with timestamp. The intended use case is a
|
||||
* multi-stage pipeline where an Entity's Java form is not needed in most stages.
|
||||
*
|
||||
* <p>For a new or updated Entity, its serialized bytes are stored along with its Datastore {@link
|
||||
* Key}. For a deleted entity, only its Datastore {@link Key} is stored, and the {@link
|
||||
* #entityProtoBytes} field is left unset.
|
||||
*
|
||||
* <p>Storing raw bytes is motivated by two factors. First, since I/O is frequent and the Java
|
||||
* objects are rarely needed in our target use case, storing raw bytes is the most efficient
|
||||
* approach. More importantly, due to our data model and our customization of {@link
|
||||
* google.registry.model.ofy.ObjectifyService ObjectifyService}, it is challenging to implement a
|
||||
* serializer for Objectify entities that preserves the value of all properties. Without such
|
||||
* serializers, Objectify entities cannot be used in a pipeline.
|
||||
*
|
||||
* <p>Objectify entities do not implement {@link Serializable}, serialization of such objects is as
|
||||
* follows:
|
||||
*
|
||||
* <ul>
|
||||
* <li>Convert an Objectify entity to a Datastore {@link Entity}: {@code
|
||||
* auditedOfy().save().toEntity(..)}
|
||||
* <li>Entity is serializable, but the more efficient approach is to convert an Entity to a
|
||||
* ProtocolBuffer ({@link com.google.storage.onestore.v3.OnestoreEntity.EntityProto}) and then
|
||||
* to raw bytes.
|
||||
* </ul>
|
||||
*
|
||||
* <p>When the first conversion above is applied to an Objectify entity, a property value in the
|
||||
* output may differ from the input in two situations:
|
||||
*
|
||||
* <ul>
|
||||
* <li>If a property is of an assign-on-persist data type, e.g., {@link
|
||||
* google.registry.model.UpdateAutoTimestamp}.
|
||||
* <li>If it is related to CommitLog management, e.g., {@link google.registry.model.EppResource
|
||||
* EppResource.revisions}.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Working around the side effects caused by our customization is difficult. Any solution would
|
||||
* likely rely on Objectify's stack of context. However, many Objectify invocations in our code base
|
||||
* are hardcoded to call the customized version of ObjectifyService, rendering Objectify's stack
|
||||
* useless.
|
||||
*
|
||||
* <p>For now, this inability to use Objectify entities in pipelines is mostly a testing problem: we
|
||||
* can not perform {@link org.apache.beam.sdk.testing.PAssert BEAM pipeline assertions} on Objectify
|
||||
* entities. {@code InitSqlTestUtils.assertContainsExactlyElementsIn} is an example of a workaround.
|
||||
*
|
||||
* <p>Note that {@link Optional java.util.Optional} is not serializable, therefore cannot be used as
|
||||
* property type in this class.
|
||||
*/
|
||||
@AutoValue
|
||||
@DeleteAfterMigration
|
||||
public abstract class VersionedEntity implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
public abstract long commitTimeMills();
|
||||
|
||||
/** The {@link Key} of the {@link Entity}. */
|
||||
public abstract Key key();
|
||||
|
||||
/** Serialized form of the {@link Entity}. This property is {@code null} for a deleted Entity. */
|
||||
@Nullable
|
||||
abstract ImmutableBytes entityProtoBytes();
|
||||
|
||||
@Memoized
|
||||
public Optional<Entity> getEntity() {
|
||||
return Optional.ofNullable(entityProtoBytes())
|
||||
.map(ImmutableBytes::getBytes)
|
||||
.map(EntityTranslator::createFromPbBytes);
|
||||
}
|
||||
|
||||
public boolean isDelete() {
|
||||
return entityProtoBytes() == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts deleted entity keys in {@code manifest} into a {@link Stream} of {@link
|
||||
* VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
|
||||
* information.
|
||||
*/
|
||||
static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
|
||||
long commitTimeMillis = manifest.getCommitTime().getMillis();
|
||||
return manifest.getDeletions().stream()
|
||||
.map(com.googlecode.objectify.Key::getRaw)
|
||||
.map(key -> newBuilder().commitTimeMills(commitTimeMillis).key(key).build());
|
||||
}
|
||||
|
||||
/* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
|
||||
static VersionedEntity fromMutation(CommitLogMutation mutation) {
|
||||
return from(
|
||||
com.googlecode.objectify.Key.create(mutation).getParent().getId(),
|
||||
mutation.getEntityProtoBytes());
|
||||
}
|
||||
|
||||
public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
|
||||
return newBuilder()
|
||||
.entityProtoBytes(entityProtoBytes)
|
||||
.key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
|
||||
.commitTimeMills(commitTimeMillis)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Builder newBuilder() {
|
||||
return new AutoValue_VersionedEntity.Builder();
|
||||
}
|
||||
|
||||
@AutoValue.Builder
|
||||
public abstract static class Builder {
|
||||
|
||||
public abstract Builder commitTimeMills(long commitTimeMillis);
|
||||
|
||||
abstract Builder entityProtoBytes(ImmutableBytes bytes);
|
||||
|
||||
public abstract Builder key(Key key);
|
||||
|
||||
public abstract VersionedEntity build();
|
||||
|
||||
Builder entityProtoBytes(byte[] bytes) {
|
||||
return entityProtoBytes(new ImmutableBytes(bytes));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a byte array and prevents it from being modified by its original owner.
|
||||
*
|
||||
* <p>While this class seems an overkill, it exists for two reasons:
|
||||
*
|
||||
* <ul>
|
||||
* <li>It is easier to override the {@link #equals} method here (for value-equivalence check)
|
||||
* than to override the AutoValue-generated {@code equals} method.
|
||||
* <li>To appease the style checker, which forbids arrays as AutoValue property.
|
||||
* </ul>
|
||||
*/
|
||||
static final class ImmutableBytes implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final byte[] bytes;
|
||||
|
||||
ImmutableBytes(byte[] bytes) {
|
||||
this.bytes = Arrays.copyOf(bytes, bytes.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the saved byte array. Invocation is restricted to trusted callers, who must not
|
||||
* modify the array.
|
||||
*/
|
||||
byte[] getBytes() {
|
||||
return bytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (!(o instanceof ImmutableBytes)) {
|
||||
return false;
|
||||
}
|
||||
ImmutableBytes that = (ImmutableBytes) o;
|
||||
// Do not use Objects.equals, which checks reference identity instead of data in array.
|
||||
return Arrays.equals(bytes, that.bytes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
// Do not use Objects.hashCode, which hashes the reference, not the data in array.
|
||||
return Arrays.hashCode(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
@javax.annotation.ParametersAreNonnullByDefault
|
||||
package google.registry.backup;
|
||||
@@ -71,7 +71,7 @@ public final class AsyncTaskEnqueuer {
|
||||
public AsyncTaskEnqueuer(
|
||||
@Named(QUEUE_ASYNC_DELETE) Queue asyncDeletePullQueue,
|
||||
@Named(QUEUE_ASYNC_HOST_RENAME) Queue asyncDnsRefreshPullQueue,
|
||||
@Config("asyncDeleteFlowMapreduceDelay") Duration asyncDeleteDelay,
|
||||
@Config("asyncDeleteDelay") Duration asyncDeleteDelay,
|
||||
CloudTasksUtils cloudTasksUtils,
|
||||
Retrier retrier) {
|
||||
this.asyncDeletePullQueue = asyncDeletePullQueue;
|
||||
|
||||
@@ -21,6 +21,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
|
||||
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
|
||||
import static google.registry.request.RequestParameters.extractBooleanParameter;
|
||||
import static google.registry.request.RequestParameters.extractIntParameter;
|
||||
import static google.registry.request.RequestParameters.extractLongParameter;
|
||||
import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
|
||||
@@ -45,6 +46,9 @@ import org.joda.time.DateTime;
|
||||
@Module
|
||||
public class BatchModule {
|
||||
|
||||
public static final String PARAM_DRY_RUN = "dryRun";
|
||||
public static final String PARAM_FAST = "fast";
|
||||
|
||||
@Provides
|
||||
@Parameter("jobName")
|
||||
static Optional<String> provideJobName(HttpServletRequest req) {
|
||||
@@ -113,9 +117,15 @@ public class BatchModule {
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(ResaveAllEppResourcesPipelineAction.PARAM_FAST)
|
||||
static Optional<Boolean> provideIsFast(HttpServletRequest req) {
|
||||
return extractOptionalBooleanParameter(req, ResaveAllEppResourcesPipelineAction.PARAM_FAST);
|
||||
@Parameter(PARAM_FAST)
|
||||
static boolean provideIsFast(HttpServletRequest req) {
|
||||
return extractBooleanParameter(req, PARAM_FAST);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(PARAM_DRY_RUN)
|
||||
static boolean provideIsDryRun(HttpServletRequest req) {
|
||||
return extractBooleanParameter(req, PARAM_DRY_RUN);
|
||||
}
|
||||
|
||||
@Provides
|
||||
|
||||
@@ -18,7 +18,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static google.registry.flows.FlowUtils.marshalWithLenientRetry;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
import static google.registry.util.ResourceUtils.readResourceUtf8;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
@@ -129,15 +128,13 @@ public class DeleteExpiredDomainsAction implements Runnable {
|
||||
logger.atInfo().log(
|
||||
"Deleting non-renewing domains with autorenew end times up through %s.", runTime);
|
||||
|
||||
// Note: in Datastore, this query is (and must be) non-transactional, and thus, is only
|
||||
// eventually consistent.
|
||||
ImmutableList<DomainBase> domainsToDelete =
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
tm().createQueryComposer(DomainBase.class)
|
||||
.where("autorenewEndTime", Comparator.LTE, runTime)
|
||||
.where("deletionTime", Comparator.EQ, END_OF_TIME)
|
||||
.list());
|
||||
tm().transact(
|
||||
() ->
|
||||
tm().createQueryComposer(DomainBase.class)
|
||||
.where("autorenewEndTime", Comparator.LTE, runTime)
|
||||
.where("deletionTime", Comparator.EQ, END_OF_TIME)
|
||||
.list());
|
||||
if (domainsToDelete.isEmpty()) {
|
||||
logger.atInfo().log("Found 0 domains to delete.");
|
||||
response.setPayload("Found 0 domains to delete.");
|
||||
|
||||
@@ -16,39 +16,30 @@ package google.registry.batch;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.batch.BatchModule.PARAM_DRY_RUN;
|
||||
import static google.registry.config.RegistryEnvironment.PRODUCTION;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.flows.poll.PollFlowUtils;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.EppResourceUtils;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex;
|
||||
import google.registry.model.poll.PollMessage;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.reporting.HistoryEntryDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.List;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
@@ -57,8 +48,8 @@ import javax.inject.Inject;
|
||||
*
|
||||
* <p>This only deletes contacts and hosts, NOT domains. To delete domains, use {@link
|
||||
* DeleteProberDataAction} and pass it the TLD(s) that the load test domains were created on. Note
|
||||
* that DeleteProberDataAction is safe enough to run in production whereas this mapreduce is not,
|
||||
* but this one does not need to be runnable in production because load testing isn't run against
|
||||
* that DeleteProberDataAction is safe enough to run in production whereas this action is not, but
|
||||
* this one does not need to be runnable in production because load testing isn't run against
|
||||
* production.
|
||||
*/
|
||||
@Action(
|
||||
@@ -79,48 +70,31 @@ public class DeleteLoadTestDataAction implements Runnable {
|
||||
private static final ImmutableSet<String> LOAD_TEST_REGISTRARS = ImmutableSet.of("proxy");
|
||||
|
||||
private final boolean isDryRun;
|
||||
private final MapreduceRunner mrRunner;
|
||||
private final Response response;
|
||||
private final Clock clock;
|
||||
|
||||
@Inject
|
||||
DeleteLoadTestDataAction(
|
||||
@Parameter(PARAM_DRY_RUN) boolean isDryRun,
|
||||
MapreduceRunner mrRunner,
|
||||
Response response,
|
||||
Clock clock) {
|
||||
this.isDryRun = isDryRun;
|
||||
this.mrRunner = mrRunner;
|
||||
this.response = response;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
// This mapreduce doesn't guarantee that foreign key relations are preserved, so isn't safe to
|
||||
// This action doesn't guarantee that foreign key relations are preserved, so isn't safe to
|
||||
// run on production. On other environments, data is fully wiped out occasionally anyway, so
|
||||
// having some broken data that isn't referred to isn't the end of the world.
|
||||
checkState(
|
||||
!RegistryEnvironment.get().equals(PRODUCTION),
|
||||
"This mapreduce is not safe to run on PRODUCTION.");
|
||||
"This action is not safe to run on PRODUCTION.");
|
||||
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Delete load test data")
|
||||
.setModuleName("backend")
|
||||
.runMapOnly(
|
||||
new DeleteLoadTestDataMapper(isDryRun),
|
||||
ImmutableList.of(
|
||||
createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
tm().transact(
|
||||
() -> {
|
||||
LOAD_TEST_REGISTRARS.forEach(this::deletePollMessages);
|
||||
tm().loadAllOfStream(ContactResource.class).forEach(this::deleteContact);
|
||||
tm().loadAllOfStream(HostResource.class).forEach(this::deleteHost);
|
||||
});
|
||||
}
|
||||
tm().transact(
|
||||
() -> {
|
||||
LOAD_TEST_REGISTRARS.forEach(this::deletePollMessages);
|
||||
tm().loadAllOfStream(ContactResource.class).forEach(this::deleteContact);
|
||||
tm().loadAllOfStream(HostResource.class).forEach(this::deleteHost);
|
||||
});
|
||||
}
|
||||
|
||||
private void deletePollMessages(String registrarId) {
|
||||
@@ -184,54 +158,4 @@ public class DeleteLoadTestDataAction implements Runnable {
|
||||
tm().delete(eppResource);
|
||||
}
|
||||
}
|
||||
|
||||
/** Provides the map method that runs for each existing contact and host entity. */
|
||||
public static class DeleteLoadTestDataMapper extends Mapper<EppResource, Void, Void> {
|
||||
|
||||
private static final long serialVersionUID = -3817710674062432694L;
|
||||
|
||||
private final boolean isDryRun;
|
||||
|
||||
public DeleteLoadTestDataMapper(boolean isDryRun) {
|
||||
this.isDryRun = isDryRun;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void map(EppResource resource) {
|
||||
if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorRegistrarId())) {
|
||||
deleteResource(resource);
|
||||
getContext()
|
||||
.incrementCounter(
|
||||
String.format("deleted %s entities", resource.getClass().getSimpleName()));
|
||||
} else {
|
||||
getContext().incrementCounter("skipped, not load test data");
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteResource(EppResource resource) {
|
||||
final Key<EppResourceIndex> eppIndex =
|
||||
Key.create(EppResourceIndex.create(Key.create(resource)));
|
||||
final Key<? extends ForeignKeyIndex<?>> fki = ForeignKeyIndex.createKey(resource);
|
||||
int numEntitiesDeleted =
|
||||
tm().transact(
|
||||
() -> {
|
||||
// This ancestor query selects all descendant entities.
|
||||
List<Key<Object>> resourceAndDependentKeys =
|
||||
auditedOfy().load().ancestor(resource).keys().list();
|
||||
ImmutableSet<Key<?>> allKeys =
|
||||
new ImmutableSet.Builder<Key<?>>()
|
||||
.add(fki)
|
||||
.add(eppIndex)
|
||||
.addAll(resourceAndDependentKeys)
|
||||
.build();
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log("Would hard-delete the following entities: %s", allKeys);
|
||||
} else {
|
||||
auditedOfy().deleteWithoutBackup().keys(allKeys);
|
||||
}
|
||||
return allKeys.size();
|
||||
});
|
||||
getContext().incrementCounter("total entities deleted", numEntitiesDeleted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,45 +17,32 @@ package google.registry.batch;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static google.registry.batch.BatchModule.PARAM_DRY_RUN;
|
||||
import static google.registry.config.RegistryEnvironment.PRODUCTION;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
|
||||
import static google.registry.model.tld.Registries.getTldsOfType;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.request.RequestParameters.PARAM_TLDS;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.dns.DnsQueue;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.model.CreateAutoTimestamp;
|
||||
import google.registry.model.EppResourceUtils;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.ForeignKeyIndex;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.model.tld.Registry.TldType;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import javax.inject.Inject;
|
||||
import org.hibernate.CacheMode;
|
||||
@@ -94,8 +81,6 @@ public class DeleteProberDataAction implements Runnable {
|
||||
*/
|
||||
private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
|
||||
|
||||
private static final DnsQueue dnsQueue = DnsQueue.create();
|
||||
|
||||
// Domains to delete must:
|
||||
// 1. Be in one of the prober TLDs
|
||||
// 2. Not be a nic domain
|
||||
@@ -115,6 +100,8 @@ public class DeleteProberDataAction implements Runnable {
|
||||
/** Number of domains to retrieve and delete per SQL transaction. */
|
||||
private static final int BATCH_SIZE = 1000;
|
||||
|
||||
@Inject DnsQueue dnsQueue;
|
||||
|
||||
@Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
|
||||
/** List of TLDs to work on. If empty - will work on all TLDs that end with .test. */
|
||||
@Inject @Parameter(PARAM_TLDS) ImmutableSet<String> tlds;
|
||||
@@ -123,8 +110,6 @@ public class DeleteProberDataAction implements Runnable {
|
||||
@Config("registryAdminClientId")
|
||||
String registryAdminRegistrarId;
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject DeleteProberDataAction() {}
|
||||
|
||||
@Override
|
||||
@@ -145,21 +130,7 @@ public class DeleteProberDataAction implements Runnable {
|
||||
"If tlds are given, they must all exist and be TEST tlds. Given: %s, not found: %s",
|
||||
tlds,
|
||||
Sets.difference(tlds, deletableTlds));
|
||||
ImmutableSet<String> proberRoidSuffixes =
|
||||
deletableTlds.stream()
|
||||
.map(tld -> Registry.get(tld).getRoidSuffix())
|
||||
.collect(toImmutableSet());
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Delete prober data")
|
||||
.setModuleName("backend")
|
||||
.runMapOnly(
|
||||
new DeleteProberDataMapper(proberRoidSuffixes, isDryRun, registryAdminRegistrarId),
|
||||
ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
runSqlJob(deletableTlds);
|
||||
}
|
||||
runSqlJob(deletableTlds);
|
||||
}
|
||||
|
||||
private void runSqlJob(ImmutableSet<String> deletableTlds) {
|
||||
@@ -231,7 +202,7 @@ public class DeleteProberDataAction implements Runnable {
|
||||
"Would soft-delete the active domain: %s (%s).",
|
||||
domain.getDomainName(), domain.getRepoId());
|
||||
} else {
|
||||
softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue);
|
||||
softDeleteDomain(domain);
|
||||
}
|
||||
softDeletedDomains.incrementAndGet();
|
||||
} else {
|
||||
@@ -280,8 +251,7 @@ public class DeleteProberDataAction implements Runnable {
|
||||
}
|
||||
|
||||
// Take a DNS queue + admin registrar id as input so that it can be called from the mapper as well
|
||||
private static void softDeleteDomain(
|
||||
DomainBase domain, String registryAdminRegistrarId, DnsQueue localDnsQueue) {
|
||||
private void softDeleteDomain(DomainBase domain) {
|
||||
DomainBase deletedDomain =
|
||||
domain.asBuilder().setDeletionTime(tm().getTransactionTime()).setStatusValues(null).build();
|
||||
DomainHistory historyEntry =
|
||||
@@ -299,119 +269,6 @@ public class DeleteProberDataAction implements Runnable {
|
||||
tm().putAllWithoutBackup(ImmutableList.of(deletedDomain, historyEntry));
|
||||
// updating foreign keys is a no-op in SQL
|
||||
updateForeignKeyIndexDeletionTime(deletedDomain);
|
||||
localDnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
|
||||
}
|
||||
|
||||
/** Provides the map method that runs for each existing DomainBase entity. */
|
||||
public static class DeleteProberDataMapper extends Mapper<Key<DomainBase>, Void, Void> {
|
||||
|
||||
private static final DnsQueue dnsQueue = DnsQueue.create();
|
||||
private static final long serialVersionUID = -7724537393697576369L;
|
||||
|
||||
private final ImmutableSet<String> proberRoidSuffixes;
|
||||
private final Boolean isDryRun;
|
||||
private final String registryAdminRegistrarId;
|
||||
|
||||
public DeleteProberDataMapper(
|
||||
ImmutableSet<String> proberRoidSuffixes,
|
||||
Boolean isDryRun,
|
||||
String registryAdminRegistrarId) {
|
||||
this.proberRoidSuffixes = proberRoidSuffixes;
|
||||
this.isDryRun = isDryRun;
|
||||
this.registryAdminRegistrarId = registryAdminRegistrarId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void map(Key<DomainBase> key) {
|
||||
try {
|
||||
String roidSuffix = Iterables.getLast(Splitter.on('-').split(key.getName()));
|
||||
if (proberRoidSuffixes.contains(roidSuffix)) {
|
||||
deleteDomain(key);
|
||||
} else {
|
||||
getContext().incrementCounter("skipped, non-prober data");
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.atSevere().withCause(t).log("Error while deleting prober data for key %s.", key);
|
||||
getContext().incrementCounter(String.format("error, kind %s", key.getKind()));
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteDomain(final Key<DomainBase> domainKey) {
|
||||
final DomainBase domain = auditedOfy().load().key(domainKey).now();
|
||||
|
||||
DateTime now = DateTime.now(UTC);
|
||||
|
||||
if (domain == null) {
|
||||
// Depending on how stale Datastore indexes are, we can get keys to resources that are
|
||||
// already deleted (e.g. by a recent previous invocation of this mapreduce). So ignore them.
|
||||
getContext().incrementCounter("already deleted");
|
||||
return;
|
||||
}
|
||||
|
||||
String domainName = domain.getDomainName();
|
||||
if (domainName.equals("nic." + domain.getTld())) {
|
||||
getContext().incrementCounter("skipped, NIC domain");
|
||||
return;
|
||||
}
|
||||
if (now.isBefore(domain.getCreationTime().plus(DOMAIN_USED_DURATION))) {
|
||||
getContext().incrementCounter("skipped, domain too new");
|
||||
return;
|
||||
}
|
||||
if (!domain.getSubordinateHosts().isEmpty()) {
|
||||
logger.atWarning().log(
|
||||
"Cannot delete domain %s (%s) because it has subordinate hosts.",
|
||||
domainName, domainKey);
|
||||
getContext().incrementCounter("skipped, had subordinate host(s)");
|
||||
return;
|
||||
}
|
||||
|
||||
// If the domain is still active, that means that the prober encountered a failure and did not
|
||||
// successfully soft-delete the domain (thus leaving its DNS entry published). We soft-delete
|
||||
// it now so that the DNS entry can be handled. The domain will then be hard-deleted the next
|
||||
// time the mapreduce is run.
|
||||
if (EppResourceUtils.isActive(domain, now)) {
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log(
|
||||
"Would soft-delete the active domain: %s (%s).", domainName, domainKey);
|
||||
} else {
|
||||
tm().transact(() -> softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue));
|
||||
}
|
||||
getContext().incrementCounter("domains soft-deleted");
|
||||
return;
|
||||
}
|
||||
// If the domain isn't active, we want to make sure it hasn't been active for "a while" before
|
||||
// deleting it. This prevents accidental double-map with the same key from immediately
|
||||
// deleting active domains
|
||||
if (now.isBefore(domain.getDeletionTime().plus(SOFT_DELETE_DELAY))) {
|
||||
getContext().incrementCounter("skipped, domain too recently soft deleted");
|
||||
return;
|
||||
}
|
||||
|
||||
final Key<EppResourceIndex> eppIndex = Key.create(EppResourceIndex.create(domainKey));
|
||||
final Key<? extends ForeignKeyIndex<?>> fki = ForeignKeyIndex.createKey(domain);
|
||||
|
||||
int entitiesDeleted =
|
||||
tm().transact(
|
||||
() -> {
|
||||
// This ancestor query selects all descendant HistoryEntries, BillingEvents,
|
||||
// PollMessages, and TLD-specific entities, as well as the domain itself.
|
||||
List<Key<Object>> domainAndDependentKeys =
|
||||
auditedOfy().load().ancestor(domainKey).keys().list();
|
||||
ImmutableSet<Key<?>> allKeys =
|
||||
new ImmutableSet.Builder<Key<?>>()
|
||||
.add(fki)
|
||||
.add(eppIndex)
|
||||
.addAll(domainAndDependentKeys)
|
||||
.build();
|
||||
if (isDryRun) {
|
||||
logger.atInfo().log("Would hard-delete the following entities: %s", allKeys);
|
||||
} else {
|
||||
auditedOfy().deleteWithoutBackup().keys(allKeys);
|
||||
}
|
||||
return allKeys.size();
|
||||
});
|
||||
getContext().incrementCounter("domains hard-deleted");
|
||||
getContext().incrementCounter("total entities hard-deleted", entitiesDeleted);
|
||||
}
|
||||
dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,35 +18,25 @@ import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static com.google.common.collect.Sets.difference;
|
||||
import static com.google.common.collect.Sets.newHashSet;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createChildEntityInput;
|
||||
import static google.registry.batch.BatchModule.PARAM_DRY_RUN;
|
||||
import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
|
||||
import static google.registry.model.domain.Period.Unit.YEARS;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
|
||||
import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
|
||||
import static google.registry.util.CollectionUtils.union;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.earliestOf;
|
||||
import static google.registry.util.DomainNameUtils.getTldFromDomainName;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Range;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.flows.domain.DomainPricingLogic;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.NullInput;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
@@ -70,13 +60,12 @@ import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.money.Money;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* A mapreduce that expands {@link Recurring} billing events into synthetic {@link OneTime} events.
|
||||
* An action that expands {@link Recurring} billing events into synthetic {@link OneTime} events.
|
||||
*
|
||||
* <p>The cursor used throughout this mapreduce (overridden if necessary using the parameter {@code
|
||||
* <p>The cursor used throughout this action (overridden if necessary using the parameter {@code
|
||||
* cursorTime}) represents the inclusive lower bound on the range of billing times that will be
|
||||
* expanded as a result of the job (the exclusive upper bound being the execution time of the job).
|
||||
*/
|
||||
@@ -87,11 +76,9 @@ import org.joda.time.DateTime;
|
||||
public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
|
||||
public static final String PARAM_CURSOR_TIME = "cursorTime";
|
||||
private static final String ERROR_COUNTER = "errors";
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject Clock clock;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
|
||||
@Inject
|
||||
@Config("jdbcBatchSize")
|
||||
@@ -108,33 +95,18 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
public void run() {
|
||||
DateTime executeTime = clock.nowUtc();
|
||||
DateTime persistedCursorTime =
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
|
||||
.orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
|
||||
.getCursorTime());
|
||||
tm().transact(
|
||||
() ->
|
||||
tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
|
||||
.orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
|
||||
.getCursorTime());
|
||||
DateTime cursorTime = cursorTimeParam.orElse(persistedCursorTime);
|
||||
checkArgument(
|
||||
cursorTime.isBefore(executeTime), "Cursor time must be earlier than execution time.");
|
||||
logger.atInfo().log(
|
||||
"Running Recurring billing event expansion for billing time range [%s, %s).",
|
||||
cursorTime, executeTime);
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Expand Recurring billing events into synthetic OneTime events.")
|
||||
.setModuleName("backend")
|
||||
.runMapreduce(
|
||||
new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
|
||||
new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
|
||||
// Add an extra shard that maps over a null recurring event (see the mapper for why).
|
||||
ImmutableList.of(
|
||||
new NullInput<>(),
|
||||
createChildEntityInput(
|
||||
ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
expandSqlBillingEventsInBatches(executeTime, cursorTime, persistedCursorTime);
|
||||
}
|
||||
expandSqlBillingEventsInBatches(executeTime, cursorTime, persistedCursorTime);
|
||||
}
|
||||
|
||||
private void expandSqlBillingEventsInBatches(
|
||||
@@ -183,7 +155,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
continue;
|
||||
}
|
||||
int billingEventsSaved =
|
||||
expandBillingEvent(recurring, executeTime, cursorTime, isDryRun);
|
||||
expandBillingEvent(
|
||||
recurring, executeTime, cursorTime, isDryRun, domainPricingLogic);
|
||||
batchBillingEventsSaved += billingEventsSaved;
|
||||
if (billingEventsSaved > 0) {
|
||||
expandedDomains.add(recurring.getTargetId());
|
||||
@@ -257,118 +230,17 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
/** Mapper to expand {@link Recurring} billing events into synthetic {@link OneTime} events. */
|
||||
public static class ExpandRecurringBillingEventsMapper
|
||||
extends Mapper<Recurring, DateTime, DateTime> {
|
||||
|
||||
private static final long serialVersionUID = 8376442755556228455L;
|
||||
|
||||
private final boolean isDryRun;
|
||||
private final DateTime cursorTime;
|
||||
private final DateTime executeTime;
|
||||
|
||||
public ExpandRecurringBillingEventsMapper(
|
||||
boolean isDryRun, DateTime cursorTime, DateTime executeTime) {
|
||||
this.isDryRun = isDryRun;
|
||||
this.cursorTime = cursorTime;
|
||||
this.executeTime = executeTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void map(final Recurring recurring) {
|
||||
// This single emit forces the reducer to run at the end of the map job, so that a mapper
|
||||
// that runs without error will advance the cursor at the end of processing (unless this was
|
||||
// a dry run, in which case the cursor should not be advanced).
|
||||
if (recurring == null) {
|
||||
emit(cursorTime, executeTime);
|
||||
return;
|
||||
}
|
||||
getContext().incrementCounter("Recurring billing events encountered");
|
||||
// Ignore any recurring billing events that have yet to apply.
|
||||
if (recurring.getEventTime().isAfter(executeTime)
|
||||
// This second case occurs when a domain is transferred or deleted before first renewal.
|
||||
|| recurring.getRecurrenceEndTime().isBefore(recurring.getEventTime())) {
|
||||
getContext().incrementCounter("Recurring billing events ignored");
|
||||
return;
|
||||
}
|
||||
int numBillingEventsSaved = 0;
|
||||
try {
|
||||
numBillingEventsSaved =
|
||||
tm().transactNew(
|
||||
() -> expandBillingEvent(recurring, executeTime, cursorTime, isDryRun));
|
||||
} catch (Throwable t) {
|
||||
getContext().incrementCounter("error: " + t.getClass().getSimpleName());
|
||||
getContext().incrementCounter(ERROR_COUNTER);
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"Error while expanding Recurring billing events for %d", recurring.getId()),
|
||||
t);
|
||||
}
|
||||
if (!isDryRun) {
|
||||
getContext().incrementCounter("Saved OneTime billing events", numBillingEventsSaved);
|
||||
} else {
|
||||
getContext()
|
||||
.incrementCounter("Generated OneTime billing events (dry run)", numBillingEventsSaved);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* "Reducer" to advance the cursor after all map jobs have been completed. The NullInput into the
|
||||
* mapper will cause the mapper to emit one timestamp pair (current cursor and execution time),
|
||||
* and the cursor will be advanced (and the timestamps logged) at the end of a successful
|
||||
* mapreduce.
|
||||
*/
|
||||
public static class ExpandRecurringBillingEventsReducer
|
||||
extends Reducer<DateTime, DateTime, Void> {
|
||||
|
||||
private final boolean isDryRun;
|
||||
private final DateTime expectedPersistedCursorTime;
|
||||
|
||||
public ExpandRecurringBillingEventsReducer(
|
||||
boolean isDryRun, DateTime expectedPersistedCursorTime) {
|
||||
this.isDryRun = isDryRun;
|
||||
this.expectedPersistedCursorTime = expectedPersistedCursorTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reduce(final DateTime cursorTime, final ReducerInput<DateTime> executionTimeInput) {
|
||||
if (getContext().getCounter(ERROR_COUNTER).getValue() > 0) {
|
||||
logger.atSevere().log(
|
||||
"One or more errors logged during recurring event expansion. Cursor will"
|
||||
+ " not be advanced.");
|
||||
return;
|
||||
}
|
||||
final DateTime executionTime = executionTimeInput.next();
|
||||
logger.atInfo().log(
|
||||
"Recurring event expansion %s complete for billing event range [%s, %s).",
|
||||
isDryRun ? "(dry run) " : "", cursorTime, executionTime);
|
||||
tm().transact(
|
||||
() -> {
|
||||
Cursor cursor =
|
||||
auditedOfy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
|
||||
DateTime currentCursorTime =
|
||||
(cursor == null ? START_OF_TIME : cursor.getCursorTime());
|
||||
if (!currentCursorTime.equals(expectedPersistedCursorTime)) {
|
||||
logger.atSevere().log(
|
||||
"Current cursor position %s does not match expected cursor position %s.",
|
||||
currentCursorTime, expectedPersistedCursorTime);
|
||||
return;
|
||||
}
|
||||
if (!isDryRun) {
|
||||
tm().put(Cursor.createGlobal(RECURRING_BILLING, executionTime));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static int expandBillingEvent(
|
||||
Recurring recurring, DateTime executeTime, DateTime cursorTime, boolean isDryRun) {
|
||||
Recurring recurring,
|
||||
DateTime executeTime,
|
||||
DateTime cursorTime,
|
||||
boolean isDryRun,
|
||||
DomainPricingLogic domainPricingLogic) {
|
||||
ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder = new ImmutableSet.Builder<>();
|
||||
final Registry tld = Registry.get(getTldFromDomainName(recurring.getTargetId()));
|
||||
|
||||
// Determine the complete set of times at which this recurring event should
|
||||
// occur (up to and including the runtime of the mapreduce).
|
||||
// occur (up to and including the runtime of the action).
|
||||
Iterable<DateTime> eventTimes =
|
||||
recurring
|
||||
.getRecurrenceTimeOfYear()
|
||||
@@ -385,14 +257,10 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
VKey.create(
|
||||
DomainBase.class, recurring.getDomainRepoId(), recurring.getParentKey().getParent());
|
||||
Iterable<OneTime> oneTimesForDomain;
|
||||
if (tm().isOfy()) {
|
||||
oneTimesForDomain = auditedOfy().load().type(OneTime.class).ancestor(domainKey.getOfyKey());
|
||||
} else {
|
||||
oneTimesForDomain =
|
||||
tm().createQueryComposer(OneTime.class)
|
||||
.where("domainRepoId", EQ, recurring.getDomainRepoId())
|
||||
.list();
|
||||
}
|
||||
oneTimesForDomain =
|
||||
tm().createQueryComposer(OneTime.class)
|
||||
.where("domainRepoId", EQ, recurring.getDomainRepoId())
|
||||
.list();
|
||||
|
||||
// Determine the billing times that already have OneTime events persisted.
|
||||
ImmutableSet<DateTime> existingBillingTimes =
|
||||
@@ -431,13 +299,16 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
historyEntriesBuilder.add(historyEntry);
|
||||
|
||||
DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
|
||||
// Determine the cost for a one-year renewal.
|
||||
Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
|
||||
|
||||
syntheticOneTimesBuilder.add(
|
||||
new OneTime.Builder()
|
||||
.setBillingTime(billingTime)
|
||||
.setRegistrarId(recurring.getRegistrarId())
|
||||
.setCost(renewCost)
|
||||
// Determine the cost for a one-year renewal.
|
||||
.setCost(
|
||||
domainPricingLogic
|
||||
.getRenewPrice(tld, recurring.getTargetId(), eventTime, 1, recurring)
|
||||
.getRenewCost())
|
||||
.setEventTime(eventTime)
|
||||
.setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
|
||||
.setParent(historyEntry)
|
||||
@@ -463,7 +334,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
|
||||
|
||||
/**
|
||||
* Filters a set of {@link DateTime}s down to event times that are in scope for a particular
|
||||
* mapreduce run, given the cursor time and the mapreduce execution time.
|
||||
* action run, given the cursor time and the action execution time.
|
||||
*/
|
||||
protected static ImmutableSet<DateTime> getBillingTimesInScope(
|
||||
Iterable<DateTime> eventTimes,
|
||||
|
||||
@@ -31,7 +31,7 @@ import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.RegistryLock;
|
||||
import google.registry.model.eppcommon.StatusValue;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarContact;
|
||||
import google.registry.model.registrar.RegistrarPoc;
|
||||
import google.registry.model.tld.RegistryLockDao;
|
||||
import google.registry.persistence.VKey;
|
||||
import google.registry.request.Action;
|
||||
@@ -120,10 +120,7 @@ public class RelockDomainAction implements Runnable {
|
||||
* for more details on retry behavior. */
|
||||
response.setStatus(SC_NO_CONTENT);
|
||||
response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
|
||||
|
||||
// nb: DomainLockUtils relies on the JPA transaction being the outermost transaction
|
||||
// if we have Datastore as the primary DB (if SQL is the primary DB, it's irrelevant)
|
||||
jpaTm().transact(() -> tm().transact(this::relockDomain));
|
||||
tm().transact(this::relockDomain);
|
||||
}
|
||||
|
||||
private void relockDomain() {
|
||||
@@ -296,8 +293,8 @@ public class RelockDomainAction implements Runnable {
|
||||
|
||||
ImmutableSet<String> registryLockEmailAddresses =
|
||||
registrar.getContacts().stream()
|
||||
.filter(RegistrarContact::isRegistryLockAllowed)
|
||||
.map(RegistrarContact::getRegistryLockEmailAddress)
|
||||
.filter(RegistrarPoc::isRegistryLockAllowed)
|
||||
.map(RegistrarPoc::getRegistryLockEmailAddress)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(toImmutableSet());
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.batch;
|
||||
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_FAST;
|
||||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* A mapreduce that re-saves all EppResources, projecting them forward to the current time.
|
||||
*
|
||||
* <p>This is useful for completing data migrations on EppResource fields that are accomplished
|
||||
* with @OnSave or @OnLoad annotations, and also guarantees that all EppResources will get fresh
|
||||
* commit logs (for backup purposes). Additionally, pending actions such as transfers or grace
|
||||
* periods that are past their effective time will be resolved.
|
||||
*
|
||||
* <p>Because there are no auth settings in the {@link Action} annotation, this command can only be
|
||||
* run internally, or by pretending to be internal by setting the X-AppEngine-QueueName header,
|
||||
* which only admin users can do.
|
||||
*
|
||||
* <p>If the <code>?fast=true</code> querystring parameter is passed, then entities that are not
|
||||
* changed by {@link EppResource#cloneProjectedAtTime} will not be re-saved. This helps prevent
|
||||
* mutation load on the DB and has the beneficial side effect of writing out smaller commit logs.
|
||||
* Note that this does NOT pick up mutations caused by migrations using the {@link
|
||||
* com.googlecode.objectify.annotation.OnLoad} annotation, so if you are running a one-off schema
|
||||
* migration, do not use fast mode. Fast mode defaults to false for this reason, but is used by the
|
||||
* monthly invocation of the mapreduce.
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/resaveAllEppResources",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
// No longer needed in SQL. Subject to future removal.
|
||||
@Deprecated
|
||||
@DeleteAfterMigration
|
||||
public class ResaveAllEppResourcesAction implements Runnable {
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
|
||||
@Inject
|
||||
@Parameter(PARAM_FAST)
|
||||
boolean isFast;
|
||||
|
||||
@Inject
|
||||
ResaveAllEppResourcesAction() {}
|
||||
|
||||
/**
|
||||
* The number of shards to run the map-only mapreduce on.
|
||||
*
|
||||
* <p>This is less than the default of 100 because we only run this action monthly and can afford
|
||||
* it being slower, but we don't want to write out lots of large commit logs in a short period of
|
||||
* time because they make the Cloud SQL migration tougher.
|
||||
*/
|
||||
private static final int NUM_SHARDS = 10;
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
mrRunner
|
||||
.setJobName("Re-save all EPP resources")
|
||||
.setModuleName("backend")
|
||||
.setDefaultMapShards(NUM_SHARDS)
|
||||
.runMapOnly(
|
||||
new ResaveAllEppResourcesActionMapper(isFast),
|
||||
ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
}
|
||||
|
||||
/** Mapper to re-save all EPP resources. */
|
||||
public static class ResaveAllEppResourcesActionMapper
|
||||
extends Mapper<Key<EppResource>, Void, Void> {
|
||||
|
||||
private static final long serialVersionUID = -7721628665138087001L;
|
||||
|
||||
private final boolean isFast;
|
||||
|
||||
ResaveAllEppResourcesActionMapper(boolean isFast) {
|
||||
this.isFast = isFast;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void map(final Key<EppResource> resourceKey) {
|
||||
boolean resaved =
|
||||
tm().transact(
|
||||
() -> {
|
||||
EppResource originalResource = auditedOfy().load().key(resourceKey).now();
|
||||
EppResource projectedResource =
|
||||
originalResource.cloneProjectedAtTime(tm().getTransactionTime());
|
||||
if (isFast && originalResource.equals(projectedResource)) {
|
||||
return false;
|
||||
} else {
|
||||
auditedOfy().save().entity(projectedResource).now();
|
||||
return true;
|
||||
}
|
||||
});
|
||||
getContext()
|
||||
.incrementCounter(
|
||||
String.format(
|
||||
"%s entities %s",
|
||||
resourceKey.getKind(), resaved ? "re-saved" : "with no changes skipped"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
package google.registry.batch;
|
||||
|
||||
import static google.registry.batch.BatchModule.PARAM_FAST;
|
||||
import static google.registry.beam.BeamUtils.createJobName;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
@@ -32,7 +33,6 @@ import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
@@ -63,8 +63,6 @@ public class ResaveAllEppResourcesPipelineAction implements Runnable {
|
||||
static final String PATH = "/_dr/task/resaveAllEppResourcesPipeline";
|
||||
static final String PIPELINE_NAME = "resave_all_epp_resources_pipeline";
|
||||
|
||||
public static final String PARAM_FAST = "fast";
|
||||
|
||||
private final String projectId;
|
||||
private final String jobRegion;
|
||||
private final String stagingBucketUrl;
|
||||
@@ -78,14 +76,14 @@ public class ResaveAllEppResourcesPipelineAction implements Runnable {
|
||||
@Config("projectId") String projectId,
|
||||
@Config("defaultJobRegion") String jobRegion,
|
||||
@Config("beamStagingBucketUrl") String stagingBucketUrl,
|
||||
@Parameter(PARAM_FAST) Optional<Boolean> fast,
|
||||
@Parameter(PARAM_FAST) boolean fast,
|
||||
Clock clock,
|
||||
Response response,
|
||||
Dataflow dataflow) {
|
||||
this.projectId = projectId;
|
||||
this.jobRegion = jobRegion;
|
||||
this.stagingBucketUrl = stagingBucketUrl;
|
||||
this.fast = fast.orElse(false);
|
||||
this.fast = fast;
|
||||
this.clock = clock;
|
||||
this.response = response;
|
||||
this.dataflow = dataflow;
|
||||
|
||||
+4
-4
@@ -32,8 +32,8 @@ import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.flows.certs.CertificateChecker;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarContact;
|
||||
import google.registry.model.registrar.RegistrarContact.Type;
|
||||
import google.registry.model.registrar.RegistrarPoc;
|
||||
import google.registry.model.registrar.RegistrarPoc.Type;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
@@ -281,9 +281,9 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
|
||||
*/
|
||||
@VisibleForTesting
|
||||
ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
|
||||
ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);
|
||||
ImmutableSortedSet<RegistrarPoc> contacts = registrar.getContactsOfType(contactType);
|
||||
ImmutableSet.Builder<InternetAddress> recipientEmails = new ImmutableSet.Builder<>();
|
||||
for (RegistrarContact contact : contacts) {
|
||||
for (RegistrarPoc contact : contacts) {
|
||||
try {
|
||||
recipientEmails.add(new InternetAddress(contact.getEmailAddress()));
|
||||
} catch (AddressException e) {
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.batch;
|
||||
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static google.registry.beam.BeamUtils.createJobName;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.api.services.dataflow.Dataflow;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Wipes out all Cloud Datastore data in a Nomulus GCP environment.
|
||||
*
|
||||
* <p>This class is created for the QA environment, where migration testing with production data
|
||||
* will happen. A regularly scheduled wipeout is a prerequisite to using production data there.
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/wipeOutDatastore",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class WipeoutDatastoreAction implements Runnable {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";
|
||||
|
||||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
private final String projectId;
|
||||
private final String jobRegion;
|
||||
private final Response response;
|
||||
private final Dataflow dataflow;
|
||||
private final String stagingBucketUrl;
|
||||
private final Clock clock;
|
||||
|
||||
@Inject
|
||||
WipeoutDatastoreAction(
|
||||
@Config("projectId") String projectId,
|
||||
@Config("defaultJobRegion") String jobRegion,
|
||||
@Config("beamStagingBucketUrl") String stagingBucketUrl,
|
||||
Clock clock,
|
||||
Response response,
|
||||
Dataflow dataflow) {
|
||||
this.projectId = projectId;
|
||||
this.jobRegion = jobRegion;
|
||||
this.stagingBucketUrl = stagingBucketUrl;
|
||||
this.clock = clock;
|
||||
this.response = response;
|
||||
this.dataflow = dataflow;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
|
||||
if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
|
||||
response.setStatus(SC_FORBIDDEN);
|
||||
response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
LaunchFlexTemplateParameter parameters =
|
||||
new LaunchFlexTemplateParameter()
|
||||
.setJobName(createJobName("bulk-delete-datastore-", clock))
|
||||
.setContainerSpecGcsPath(
|
||||
String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
|
||||
.setParameters(
|
||||
ImmutableMap.of(
|
||||
"kindsToDelete",
|
||||
"*",
|
||||
"registryEnvironment",
|
||||
RegistryEnvironment.get().name()));
|
||||
LaunchFlexTemplateResponse launchResponse =
|
||||
dataflow
|
||||
.projects()
|
||||
.locations()
|
||||
.flexTemplates()
|
||||
.launch(
|
||||
projectId,
|
||||
jobRegion,
|
||||
new LaunchFlexTemplateRequest().setLaunchParameter(parameters))
|
||||
.execute();
|
||||
response.setStatus(SC_OK);
|
||||
response.setPayload("Launched " + launchResponse.getJob().getName());
|
||||
} catch (Exception e) {
|
||||
String msg = String.format("Failed to launch %s.", PIPELINE_NAME);
|
||||
logger.atSevere().withCause(e).log(msg);
|
||||
response.setStatus(SC_INTERNAL_SERVER_ERROR);
|
||||
response.setPayload(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,336 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.datastore;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
|
||||
import static org.apache.beam.sdk.values.TypeDescriptors.strings;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.datastore.v1.Entity;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
|
||||
import org.apache.beam.sdk.io.gcp.datastore.DatastoreIO;
|
||||
import org.apache.beam.sdk.options.Default;
|
||||
import org.apache.beam.sdk.options.Description;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.options.Validation;
|
||||
import org.apache.beam.sdk.transforms.Create;
|
||||
import org.apache.beam.sdk.transforms.DoFn;
|
||||
import org.apache.beam.sdk.transforms.GroupByKey;
|
||||
import org.apache.beam.sdk.transforms.MapElements;
|
||||
import org.apache.beam.sdk.transforms.PTransform;
|
||||
import org.apache.beam.sdk.transforms.ParDo;
|
||||
import org.apache.beam.sdk.transforms.Reshuffle;
|
||||
import org.apache.beam.sdk.transforms.View;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PBegin;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.apache.beam.sdk.values.PCollectionTuple;
|
||||
import org.apache.beam.sdk.values.PCollectionView;
|
||||
import org.apache.beam.sdk.values.TupleTag;
|
||||
import org.apache.beam.sdk.values.TupleTagList;
|
||||
|
||||
/**
|
||||
* A BEAM pipeline that deletes Datastore entities in bulk.
|
||||
*
|
||||
* <p>This pipeline provides an alternative to the <a
|
||||
* href="https://cloud.google.com/datastore/docs/bulk-delete">GCP builtin template</a> that performs
|
||||
* the same task. It solves the following performance and usability problems in the builtin
|
||||
* template:
|
||||
*
|
||||
* <ul>
|
||||
* <li>When deleting all data (by using the {@code select __key__} or {@code select *} queries),
|
||||
* the builtin template cannot parallelize the query, therefore has to query with a single
|
||||
* worker.
|
||||
* <li>When deleting all data, the builtin template also attempts to delete Datastore internal
|
||||
* tables which would cause permission-denied errors, which in turn MAY cause the pipeline to
|
||||
* abort before all data has been deleted.
|
||||
* <li>With the builtin template, it is possible to delete multiple entity types in one pipeline
|
||||
* ONLY if the user can come up with a single literal query that covers all of them. This is
|
||||
* not the case with most Nomulus entity types.
|
||||
* </ul>
|
||||
*
|
||||
* <p>A user of this pipeline must specify the types of entities to delete using the {@code
|
||||
* --kindsToDelete} command line argument. To delete specific entity types, give a comma-separated
|
||||
* string of their kind names; to delete all data, give {@code "*"}.
|
||||
*
|
||||
* <p>When deleting all data, it is recommended for the user to specify the number of user entity
|
||||
* types in the Datastore using the {@code --numOfKindsHint} argument. If the default value for this
|
||||
* parameter is too low, performance will suffer.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class BulkDeleteDatastorePipeline {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
// This tool is not for use in our critical projects.
|
||||
private static final ImmutableSet<String> FORBIDDEN_PROJECTS =
|
||||
ImmutableSet.of("domain-registry", "domain-registry-sandbox");
|
||||
|
||||
private final BulkDeletePipelineOptions options;
|
||||
|
||||
BulkDeleteDatastorePipeline(BulkDeletePipelineOptions options) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
Pipeline pipeline = Pipeline.create(options);
|
||||
setupPipeline(pipeline);
|
||||
pipeline.run();
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation") // org.apache.beam.sdk.transforms.Reshuffle
|
||||
private void setupPipeline(Pipeline pipeline) {
|
||||
checkState(
|
||||
!FORBIDDEN_PROJECTS.contains(options.getProject()),
|
||||
"Bulk delete is forbidden in %s",
|
||||
options.getProject());
|
||||
|
||||
// Pre-allocated tags to label entities by kind. In the case of delete-all, we must use a guess.
|
||||
TupleTagList deletionTags;
|
||||
PCollection<String> kindsToDelete;
|
||||
|
||||
if (options.getKindsToDelete().equals("*")) {
|
||||
deletionTags = getDeletionTags(options.getNumOfKindsHint());
|
||||
kindsToDelete =
|
||||
pipeline.apply("DiscoverEntityKinds", discoverEntityKinds(options.getProject()));
|
||||
} else {
|
||||
ImmutableList<String> kindsToDeleteParam = parseKindsToDelete(options);
|
||||
checkState(
|
||||
!kindsToDeleteParam.contains("*"),
|
||||
"The --kindsToDelete argument should not contain both '*' and other kinds.");
|
||||
deletionTags = getDeletionTags(kindsToDeleteParam.size());
|
||||
kindsToDelete = pipeline.apply("UseProvidedKinds", Create.of(kindsToDeleteParam));
|
||||
}
|
||||
|
||||
// Map each kind to a tag. The "SplitByKind" stage below will group entities by kind using
|
||||
// this mapping. In practice, this has been effective at avoiding entity group contentions.
|
||||
PCollectionView<Map<String, TupleTag<Entity>>> kindToTagMapping =
|
||||
mapKindsToDeletionTags(kindsToDelete, deletionTags).apply("GetKindsToTagMap", View.asMap());
|
||||
|
||||
PCollectionTuple entities =
|
||||
kindsToDelete
|
||||
.apply("GenerateQueries", ParDo.of(new GenerateQueries()))
|
||||
.apply("ReadEntities", DatastoreV1.read().withProjectId(options.getProject()))
|
||||
.apply(
|
||||
"SplitByKind",
|
||||
ParDo.of(new SplitEntities(kindToTagMapping))
|
||||
.withSideInputs(kindToTagMapping)
|
||||
.withOutputTags(getOneDeletionTag("placeholder"), deletionTags));
|
||||
|
||||
for (TupleTag<?> tag : deletionTags.getAll()) {
|
||||
entities
|
||||
.get((TupleTag<Entity>) tag)
|
||||
// Reshuffle calls GroupByKey which is one way to trigger load rebalance in the pipeline.
|
||||
// Using the deprecated "Reshuffle" for convenience given the short life of this tool.
|
||||
.apply("RebalanceLoad", Reshuffle.viaRandomKey())
|
||||
.apply(
|
||||
"DeleteEntities_" + tag.getId(),
|
||||
DatastoreIO.v1().deleteEntity().withProjectId(options.getProject()));
|
||||
}
|
||||
}
|
||||
|
||||
private static String toKeyOnlyQueryForKind(String kind) {
|
||||
return "select __key__ from `" + kind + "`";
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link TupleTag} that retains the generic type parameter and may be used in a
|
||||
* multi-output {@link ParDo} (e.g. {@link SplitEntities}).
|
||||
*
|
||||
* <p>This method is NOT needed in tests when creating tags for assertions. Simply create them
|
||||
* with {@code new TupleTag<Entity>(String)}.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static TupleTag<Entity> getOneDeletionTag(String id) {
|
||||
// The trailing {} is needed to retain generic param type.
|
||||
return new TupleTag<Entity>(id) {};
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static ImmutableList<String> parseKindsToDelete(BulkDeletePipelineOptions options) {
|
||||
return ImmutableList.copyOf(
|
||||
Splitter.on(",").omitEmptyStrings().trimResults().split(options.getKindsToDelete().trim()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list of {@code n} {@link TupleTag TupleTags} numbered from {@code 0} to {@code n-1}.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static TupleTagList getDeletionTags(int n) {
|
||||
ImmutableList.Builder<TupleTag<?>> builder = new ImmutableList.Builder<>();
|
||||
for (int i = 0; i < n; i++) {
|
||||
builder.add(getOneDeletionTag(String.valueOf(i)));
|
||||
}
|
||||
return TupleTagList.of(builder.build());
|
||||
}
|
||||
|
||||
/** Returns a {@link PTransform} that finds all entity kinds in Datastore. */
|
||||
@VisibleForTesting
|
||||
static PTransform<PBegin, PCollection<String>> discoverEntityKinds(String project) {
|
||||
return new PTransform<PBegin, PCollection<String>>() {
|
||||
@Override
|
||||
public PCollection<String> expand(PBegin input) {
|
||||
// Use the __kind__ table to discover entity kinds. Data in the more informational
|
||||
// __Stat_Kind__ table may be up to 48-hour stale.
|
||||
return input
|
||||
.apply(
|
||||
"LoadEntityMetaData",
|
||||
DatastoreIO.v1()
|
||||
.read()
|
||||
.withProjectId(project)
|
||||
.withLiteralGqlQuery("select * from __kind__"))
|
||||
.apply(
|
||||
"GetKindNames",
|
||||
ParDo.of(
|
||||
new DoFn<Entity, String>() {
|
||||
@ProcessElement
|
||||
public void processElement(
|
||||
@Element Entity entity, OutputReceiver<String> out) {
|
||||
String kind = entity.getKey().getPath(0).getName();
|
||||
if (kind.startsWith("_")) {
|
||||
return;
|
||||
}
|
||||
out.output(kind);
|
||||
}
|
||||
}));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static PCollection<KV<String, TupleTag<Entity>>> mapKindsToDeletionTags(
|
||||
PCollection<String> kinds, TupleTagList tags) {
|
||||
// The first two stages send all strings in the 'kinds' PCollection to one worker which
|
||||
// performs the mapping in the last stage.
|
||||
return kinds
|
||||
.apply(
|
||||
"AssignSingletonKeyToKinds",
|
||||
MapElements.into(kvs(strings(), strings())).via(kind -> KV.of("", kind)))
|
||||
.apply("GatherKindsIntoCollection", GroupByKey.create())
|
||||
.apply("MapKindsToTag", ParDo.of(new MapKindsToTags(tags)));
|
||||
}
|
||||
|
||||
/** Transforms each {@code kind} string into a Datastore query for that kind. */
|
||||
@VisibleForTesting
|
||||
static class GenerateQueries extends DoFn<String, String> {
|
||||
@ProcessElement
|
||||
public void processElement(@Element String kind, OutputReceiver<String> out) {
|
||||
out.output(toKeyOnlyQueryForKind(kind));
|
||||
}
|
||||
}
|
||||
|
||||
private static class MapKindsToTags
|
||||
extends DoFn<KV<String, Iterable<String>>, KV<String, TupleTag<Entity>>> {
|
||||
private final TupleTagList tupleTags;
|
||||
|
||||
MapKindsToTags(TupleTagList tupleTags) {
|
||||
this.tupleTags = tupleTags;
|
||||
}
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(
|
||||
@Element KV<String, Iterable<String>> kv,
|
||||
OutputReceiver<KV<String, TupleTag<Entity>>> out) {
|
||||
// Sort kinds so that mapping is deterministic.
|
||||
ImmutableSortedSet<String> sortedKinds = ImmutableSortedSet.copyOf(kv.getValue());
|
||||
Iterator<String> kinds = sortedKinds.iterator();
|
||||
Iterator<TupleTag<?>> tags = tupleTags.getAll().iterator();
|
||||
|
||||
while (kinds.hasNext() && tags.hasNext()) {
|
||||
out.output(KV.of(kinds.next(), (TupleTag<Entity>) tags.next()));
|
||||
}
|
||||
|
||||
if (kinds.hasNext()) {
|
||||
logger.atWarning().log(
|
||||
"There are more kinds to delete (%s) than our estimate (%s). "
|
||||
+ "Performance may suffer.",
|
||||
sortedKinds.size(), tupleTags.size());
|
||||
}
|
||||
// Round robin assignment so that mapping is deterministic
|
||||
while (kinds.hasNext()) {
|
||||
tags = tupleTags.getAll().iterator();
|
||||
while (kinds.hasNext() && tags.hasNext()) {
|
||||
out.output(KV.of(kinds.next(), (TupleTag<Entity>) tags.next()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* {@link DoFn} that splits one {@link PCollection} of mixed kinds into multiple single-kind
|
||||
* {@code PCollections}.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static class SplitEntities extends DoFn<Entity, Entity> {
|
||||
private final PCollectionView<Map<String, TupleTag<Entity>>> kindToTagMapping;
|
||||
|
||||
SplitEntities(PCollectionView<Map<String, TupleTag<Entity>>> kindToTagMapping) {
|
||||
super();
|
||||
this.kindToTagMapping = kindToTagMapping;
|
||||
}
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(ProcessContext context) {
|
||||
Entity entity = context.element();
|
||||
com.google.datastore.v1.Key entityKey = entity.getKey();
|
||||
String kind = entityKey.getPath(entityKey.getPathCount() - 1).getKind();
|
||||
TupleTag<Entity> tag = context.sideInput(kindToTagMapping).get(kind);
|
||||
context.output(tag, entity);
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
BulkDeletePipelineOptions options =
|
||||
PipelineOptionsFactory.fromArgs(args).withValidation().as(BulkDeletePipelineOptions.class);
|
||||
BulkDeleteDatastorePipeline pipeline = new BulkDeleteDatastorePipeline(options);
|
||||
pipeline.run();
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
public interface BulkDeletePipelineOptions extends GcpOptions {
|
||||
|
||||
@Description("The Registry environment.")
|
||||
RegistryEnvironment getRegistryEnvironment();
|
||||
|
||||
void setRegistryEnvironment(RegistryEnvironment environment);
|
||||
|
||||
@Description(
|
||||
"The Datastore KINDs to be deleted. The format may be:\n"
|
||||
+ "\t- The list of kinds to be deleted as a comma-separated string, or\n"
|
||||
+ "\t- '*', which causes all kinds to be deleted.")
|
||||
@Validation.Required
|
||||
String getKindsToDelete();
|
||||
|
||||
void setKindsToDelete(String kinds);
|
||||
|
||||
@Description(
|
||||
"An estimate of the number of KINDs to be deleted. "
|
||||
+ "This is recommended if --kindsToDelete is '*' and the default value is too low.")
|
||||
@Default.Integer(30)
|
||||
int getNumOfKindsHint();
|
||||
|
||||
void setNumOfKindsHint(int numOfKindsHint);
|
||||
}
|
||||
}
|
||||
@@ -1,768 +0,0 @@
|
||||
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This class is adapted from the Apache BEAM SDK. The original license may
|
||||
// be found at <a href="https://github.com/apache/beam/blob/master/LICENSE">
|
||||
// this link</a>.
|
||||
|
||||
package google.registry.beam.datastore;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static com.google.datastore.v1.PropertyFilter.Operator.EQUAL;
|
||||
import static com.google.datastore.v1.PropertyOrder.Direction.DESCENDING;
|
||||
import static com.google.datastore.v1.QueryResultBatch.MoreResultsType.NOT_FINISHED;
|
||||
import static com.google.datastore.v1.client.DatastoreHelper.makeAndFilter;
|
||||
import static com.google.datastore.v1.client.DatastoreHelper.makeFilter;
|
||||
import static com.google.datastore.v1.client.DatastoreHelper.makeOrder;
|
||||
import static com.google.datastore.v1.client.DatastoreHelper.makeValue;
|
||||
|
||||
import com.google.api.client.http.HttpRequestInitializer;
|
||||
import com.google.auth.Credentials;
|
||||
import com.google.auth.http.HttpCredentialsAdapter;
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.datastore.v1.Entity;
|
||||
import com.google.datastore.v1.EntityResult;
|
||||
import com.google.datastore.v1.GqlQuery;
|
||||
import com.google.datastore.v1.PartitionId;
|
||||
import com.google.datastore.v1.Query;
|
||||
import com.google.datastore.v1.QueryResultBatch;
|
||||
import com.google.datastore.v1.RunQueryRequest;
|
||||
import com.google.datastore.v1.RunQueryResponse;
|
||||
import com.google.datastore.v1.client.Datastore;
|
||||
import com.google.datastore.v1.client.DatastoreException;
|
||||
import com.google.datastore.v1.client.DatastoreFactory;
|
||||
import com.google.datastore.v1.client.DatastoreHelper;
|
||||
import com.google.datastore.v1.client.DatastoreOptions;
|
||||
import com.google.datastore.v1.client.QuerySplitter;
|
||||
import com.google.protobuf.Int32Value;
|
||||
import com.google.rpc.Code;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import javax.annotation.Nullable;
|
||||
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
|
||||
import org.apache.beam.sdk.extensions.gcp.util.RetryHttpRequestInitializer;
|
||||
import org.apache.beam.sdk.metrics.Counter;
|
||||
import org.apache.beam.sdk.metrics.Metrics;
|
||||
import org.apache.beam.sdk.options.PipelineOptions;
|
||||
import org.apache.beam.sdk.transforms.DoFn;
|
||||
import org.apache.beam.sdk.transforms.PTransform;
|
||||
import org.apache.beam.sdk.transforms.ParDo;
|
||||
import org.apache.beam.sdk.transforms.Reshuffle;
|
||||
import org.apache.beam.sdk.transforms.display.DisplayData;
|
||||
import org.apache.beam.sdk.transforms.display.HasDisplayData;
|
||||
import org.apache.beam.sdk.util.BackOff;
|
||||
import org.apache.beam.sdk.util.BackOffUtils;
|
||||
import org.apache.beam.sdk.util.FluentBackoff;
|
||||
import org.apache.beam.sdk.util.Sleeper;
|
||||
import org.apache.beam.sdk.values.KV;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/**
|
||||
* Contains an adaptation of {@link org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read}. See
|
||||
* {@link MultiRead} for details.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class DatastoreV1 {
|
||||
|
||||
// A package-private constructor to prevent direct instantiation from outside of this package
|
||||
DatastoreV1() {}
|
||||
|
||||
/**
|
||||
* Non-retryable errors. See https://cloud.google.com/datastore/docs/concepts/errors#Error_Codes .
|
||||
*/
|
||||
private static final ImmutableSet<Code> NON_RETRYABLE_ERRORS =
|
||||
ImmutableSet.of(
|
||||
Code.FAILED_PRECONDITION,
|
||||
Code.INVALID_ARGUMENT,
|
||||
Code.PERMISSION_DENIED,
|
||||
Code.UNAUTHENTICATED);
|
||||
|
||||
/**
|
||||
* Returns an empty {@link MultiRead} builder. Configure the source {@code projectId}, {@code
|
||||
* query}, and optionally {@code namespace} and {@code numQuerySplits} using {@link
|
||||
* MultiRead#withProjectId}, {@link MultiRead#withNamespace}, {@link
|
||||
* MultiRead#withNumQuerySplits}.
|
||||
*/
|
||||
public static MultiRead read() {
|
||||
return new AutoValue_DatastoreV1_MultiRead.Builder().setNumQuerySplits(0).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* A {@link PTransform} that executes every Cloud SQL queries in a {@link PCollection } and reads
|
||||
* their result rows as {@code Entity} objects.
|
||||
*
|
||||
* <p>This class is adapted from {@link org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read}. It
|
||||
* uses literal GQL queries in the input {@link PCollection} instead of a constant query provided
|
||||
* to the builder. Only the {@link #expand} method is modified from the original. Everything else
|
||||
* including comments have been copied verbatim.
|
||||
*/
|
||||
@AutoValue
|
||||
public abstract static class MultiRead
|
||||
extends PTransform<PCollection<String>, PCollection<Entity>> {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
/** An upper bound on the number of splits for a query. */
|
||||
public static final int NUM_QUERY_SPLITS_MAX = 50000;
|
||||
|
||||
/** A lower bound on the number of splits for a query. */
|
||||
static final int NUM_QUERY_SPLITS_MIN = 12;
|
||||
|
||||
/** Default bundle size of 64MB. */
|
||||
static final long DEFAULT_BUNDLE_SIZE_BYTES = 64L * 1024L * 1024L;
|
||||
|
||||
/**
|
||||
* Maximum number of results to request per query.
|
||||
*
|
||||
* <p>Must be set, or it may result in an I/O error when querying Cloud Datastore.
|
||||
*/
|
||||
static final int QUERY_BATCH_LIMIT = 500;
|
||||
|
||||
public abstract @Nullable String getProjectId();
|
||||
|
||||
public abstract @Nullable String getNamespace();
|
||||
|
||||
public abstract int getNumQuerySplits();
|
||||
|
||||
public abstract @Nullable String getLocalhost();
|
||||
|
||||
@Override
|
||||
public abstract String toString();
|
||||
|
||||
abstract Builder toBuilder();
|
||||
|
||||
@AutoValue.Builder
|
||||
abstract static class Builder {
|
||||
abstract Builder setProjectId(String projectId);
|
||||
|
||||
abstract Builder setNamespace(String namespace);
|
||||
|
||||
abstract Builder setNumQuerySplits(int numQuerySplits);
|
||||
|
||||
abstract Builder setLocalhost(String localhost);
|
||||
|
||||
abstract MultiRead build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the number of splits to be performed on the given query by querying the estimated
|
||||
* size from Cloud Datastore.
|
||||
*/
|
||||
static int getEstimatedNumSplits(Datastore datastore, Query query, @Nullable String namespace) {
|
||||
int numSplits;
|
||||
try {
|
||||
long estimatedSizeBytes = getEstimatedSizeBytes(datastore, query, namespace);
|
||||
logger.atInfo().log("Estimated size for the query is %d bytes.", estimatedSizeBytes);
|
||||
numSplits =
|
||||
(int)
|
||||
Math.min(
|
||||
NUM_QUERY_SPLITS_MAX,
|
||||
Math.round(((double) estimatedSizeBytes) / DEFAULT_BUNDLE_SIZE_BYTES));
|
||||
} catch (Exception e) {
|
||||
logger.atWarning().withCause(e).log(
|
||||
"Failed the fetch estimatedSizeBytes for query: %s", query);
|
||||
// Fallback in case estimated size is unavailable.
|
||||
numSplits = NUM_QUERY_SPLITS_MIN;
|
||||
}
|
||||
return Math.max(numSplits, NUM_QUERY_SPLITS_MIN);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cloud Datastore system tables with statistics are periodically updated. This method fetches
|
||||
* the latest timestamp (in microseconds) of statistics update using the {@code __Stat_Total__}
|
||||
* table.
|
||||
*/
|
||||
private static long queryLatestStatisticsTimestamp(
|
||||
Datastore datastore, @Nullable String namespace) throws DatastoreException {
|
||||
Query.Builder query = Query.newBuilder();
|
||||
// Note: namespace either being null or empty represents the default namespace, in which
|
||||
// case we treat it as not provided by the user.
|
||||
if (Strings.isNullOrEmpty(namespace)) {
|
||||
query.addKindBuilder().setName("__Stat_Total__");
|
||||
} else {
|
||||
query.addKindBuilder().setName("__Stat_Ns_Total__");
|
||||
}
|
||||
query.addOrder(makeOrder("timestamp", DESCENDING));
|
||||
query.setLimit(Int32Value.newBuilder().setValue(1));
|
||||
RunQueryRequest request = makeRequest(query.build(), namespace);
|
||||
|
||||
RunQueryResponse response = datastore.runQuery(request);
|
||||
QueryResultBatch batch = response.getBatch();
|
||||
if (batch.getEntityResultsCount() == 0) {
|
||||
throw new NoSuchElementException("Datastore total statistics unavailable");
|
||||
}
|
||||
Entity entity = batch.getEntityResults(0).getEntity();
|
||||
return entity.getProperties().get("timestamp").getTimestampValue().getSeconds() * 1000000;
|
||||
}
|
||||
|
||||
/** Retrieve latest table statistics for a given kind, namespace, and datastore. */
|
||||
private static Entity getLatestTableStats(
|
||||
String ourKind, @Nullable String namespace, Datastore datastore) throws DatastoreException {
|
||||
long latestTimestamp = queryLatestStatisticsTimestamp(datastore, namespace);
|
||||
logger.atInfo().log("Latest stats timestamp for kind %s is %s.", ourKind, latestTimestamp);
|
||||
|
||||
Query.Builder queryBuilder = Query.newBuilder();
|
||||
if (Strings.isNullOrEmpty(namespace)) {
|
||||
queryBuilder.addKindBuilder().setName("__Stat_Kind__");
|
||||
} else {
|
||||
queryBuilder.addKindBuilder().setName("__Stat_Ns_Kind__");
|
||||
}
|
||||
|
||||
queryBuilder.setFilter(
|
||||
makeAndFilter(
|
||||
makeFilter("kind_name", EQUAL, makeValue(ourKind).build()).build(),
|
||||
makeFilter("timestamp", EQUAL, makeValue(latestTimestamp).build()).build()));
|
||||
|
||||
RunQueryRequest request = makeRequest(queryBuilder.build(), namespace);
|
||||
|
||||
long now = System.currentTimeMillis();
|
||||
RunQueryResponse response = datastore.runQuery(request);
|
||||
logger.atFine().log(
|
||||
"Query for per-kind statistics took %d ms.", System.currentTimeMillis() - now);
|
||||
|
||||
QueryResultBatch batch = response.getBatch();
|
||||
if (batch.getEntityResultsCount() == 0) {
|
||||
throw new NoSuchElementException(
|
||||
"Datastore statistics for kind " + ourKind + " unavailable");
|
||||
}
|
||||
return batch.getEntityResults(0).getEntity();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the estimated size of the data returned by the given query.
|
||||
*
|
||||
* <p>Cloud Datastore provides no way to get a good estimate of how large the result of a query
|
||||
* entity kind being queried, using the __Stat_Kind__ system table, assuming exactly 1 kind is
|
||||
* specified in the query.
|
||||
*
|
||||
* <p>See https://cloud.google.com/datastore/docs/concepts/stats.
|
||||
*/
|
||||
static long getEstimatedSizeBytes(Datastore datastore, Query query, @Nullable String namespace)
|
||||
throws DatastoreException {
|
||||
String ourKind = query.getKind(0).getName();
|
||||
Entity entity = getLatestTableStats(ourKind, namespace, datastore);
|
||||
return entity.getProperties().get("entity_bytes").getIntegerValue();
|
||||
}
|
||||
|
||||
private static PartitionId.Builder forNamespace(@Nullable String namespace) {
|
||||
PartitionId.Builder partitionBuilder = PartitionId.newBuilder();
|
||||
// Namespace either being null or empty represents the default namespace.
|
||||
// Datastore Client libraries expect users to not set the namespace proto field in
|
||||
// either of these cases.
|
||||
if (!Strings.isNullOrEmpty(namespace)) {
|
||||
partitionBuilder.setNamespaceId(namespace);
|
||||
}
|
||||
return partitionBuilder;
|
||||
}
|
||||
|
||||
/** Builds a {@link RunQueryRequest} from the {@code query} and {@code namespace}. */
|
||||
static RunQueryRequest makeRequest(Query query, @Nullable String namespace) {
|
||||
return RunQueryRequest.newBuilder()
|
||||
.setQuery(query)
|
||||
.setPartitionId(forNamespace(namespace))
|
||||
.build();
|
||||
}
|
||||
|
||||
/** Builds a {@link RunQueryRequest} from the {@code GqlQuery} and {@code namespace}. */
|
||||
private static RunQueryRequest makeRequest(GqlQuery gqlQuery, @Nullable String namespace) {
|
||||
return RunQueryRequest.newBuilder()
|
||||
.setGqlQuery(gqlQuery)
|
||||
.setPartitionId(forNamespace(namespace))
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* A helper function to get the split queries, taking into account the optional {@code
|
||||
* namespace}.
|
||||
*/
|
||||
private static List<Query> splitQuery(
|
||||
Query query,
|
||||
@Nullable String namespace,
|
||||
Datastore datastore,
|
||||
QuerySplitter querySplitter,
|
||||
int numSplits)
|
||||
throws DatastoreException {
|
||||
// If namespace is set, include it in the split request so splits are calculated accordingly.
|
||||
return querySplitter.getSplits(query, forNamespace(namespace).build(), numSplits, datastore);
|
||||
}
|
||||
|
||||
/**
|
||||
* Translates a Cloud Datastore gql query string to {@link Query}.
|
||||
*
|
||||
* <p>Currently, the only way to translate a gql query string to a Query is to run the query
|
||||
* against Cloud Datastore and extract the {@code Query} from the response. To prevent reading
|
||||
* any data, we set the {@code LIMIT} to 0 but if the gql query already has a limit set, we
|
||||
* catch the exception with {@code INVALID_ARGUMENT} error code and retry the translation
|
||||
* without the zero limit.
|
||||
*
|
||||
* <p>Note: This may result in reading actual data from Cloud Datastore but the service has a
|
||||
* cap on the number of entities returned for a single rpc request, so this should not be a
|
||||
* problem in practice.
|
||||
*/
|
||||
private static Query translateGqlQueryWithLimitCheck(
|
||||
String gql, Datastore datastore, String namespace) throws DatastoreException {
|
||||
String gqlQueryWithZeroLimit = gql + " LIMIT 0";
|
||||
try {
|
||||
Query translatedQuery = translateGqlQuery(gqlQueryWithZeroLimit, datastore, namespace);
|
||||
// Clear the limit that we set.
|
||||
return translatedQuery.toBuilder().clearLimit().build();
|
||||
} catch (DatastoreException e) {
|
||||
// Note: There is no specific error code or message to detect if the query already has a
|
||||
// limit, so we just check for INVALID_ARGUMENT and assume that that the query might have
|
||||
// a limit already set.
|
||||
if (e.getCode() == Code.INVALID_ARGUMENT) {
|
||||
logger.atWarning().log(
|
||||
"Failed to translate Gql query '%s': %s", gqlQueryWithZeroLimit, e.getMessage());
|
||||
logger.atWarning().log(
|
||||
"User query might have a limit already set, so trying without zero limit.");
|
||||
// Retry without the zero limit.
|
||||
return translateGqlQuery(gql, datastore, namespace);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Translates a gql query string to {@link Query}. */
|
||||
private static Query translateGqlQuery(String gql, Datastore datastore, String namespace)
|
||||
throws DatastoreException {
|
||||
logger.atInfo().log("Translating gql %s", gql);
|
||||
GqlQuery gqlQuery = GqlQuery.newBuilder().setQueryString(gql).setAllowLiterals(true).build();
|
||||
RunQueryRequest req = makeRequest(gqlQuery, namespace);
|
||||
return datastore.runQuery(req).getQuery();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new {@link MultiRead} that reads from the Cloud Datastore for the specified
|
||||
* project.
|
||||
*/
|
||||
public MultiRead withProjectId(String projectId) {
|
||||
checkArgument(projectId != null, "projectId can not be null");
|
||||
return toBuilder().setProjectId(projectId).build();
|
||||
}
|
||||
|
||||
/** Returns a new {@link MultiRead} that reads from the given namespace. */
|
||||
public MultiRead withNamespace(String namespace) {
|
||||
return toBuilder().setNamespace(namespace).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new {@link MultiRead} that reads by splitting the given {@code query} into {@code
|
||||
* numQuerySplits}.
|
||||
*
|
||||
* <p>The semantics for the query splitting is defined below:
|
||||
*
|
||||
* <ul>
|
||||
* <li>Any value less than or equal to 0 will be ignored, and the number of splits will be
|
||||
* chosen dynamically at runtime based on the query data size.
|
||||
* <li>Any value greater than {@link MultiRead#NUM_QUERY_SPLITS_MAX} will be capped at {@code
|
||||
* NUM_QUERY_SPLITS_MAX}.
|
||||
* <li>If the {@code query} has a user limit set, then {@code numQuerySplits} will be ignored
|
||||
* and no split will be performed.
|
||||
* <li>Under certain cases Cloud Datastore is unable to split query to the requested number of
|
||||
* splits. In such cases we just use whatever the Cloud Datastore returns.
|
||||
* </ul>
|
||||
*/
|
||||
public MultiRead withNumQuerySplits(int numQuerySplits) {
|
||||
return toBuilder()
|
||||
.setNumQuerySplits(Math.min(Math.max(numQuerySplits, 0), NUM_QUERY_SPLITS_MAX))
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new {@link MultiRead} that reads from a Datastore Emulator running at the given
|
||||
* localhost address.
|
||||
*/
|
||||
public MultiRead withLocalhost(String localhost) {
|
||||
return toBuilder().setLocalhost(localhost).build();
|
||||
}
|
||||
|
||||
/** Returns Number of entities available for reading. */
|
||||
public long getNumEntities(
|
||||
PipelineOptions options, String ourKind, @Nullable String namespace) {
|
||||
try {
|
||||
V1Options v1Options = V1Options.from(getProjectId(), getNamespace(), getLocalhost());
|
||||
V1DatastoreFactory datastoreFactory = new V1DatastoreFactory();
|
||||
Datastore datastore =
|
||||
datastoreFactory.getDatastore(
|
||||
options, v1Options.getProjectId(), v1Options.getLocalhost());
|
||||
|
||||
Entity entity = getLatestTableStats(ourKind, namespace, datastore);
|
||||
return entity.getProperties().get("count").getIntegerValue();
|
||||
} catch (Exception e) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PCollection<Entity> expand(PCollection<String> gqlQueries) {
|
||||
checkArgument(getProjectId() != null, "projectId cannot be null");
|
||||
|
||||
V1Options v1Options = V1Options.from(getProjectId(), getNamespace(), getLocalhost());
|
||||
|
||||
/*
|
||||
* This composite transform involves the following steps:
|
||||
* 1. Apply a {@link ParDo} that translates each query in {@code gqlQueries} into a {@code
|
||||
* query}.
|
||||
*
|
||||
* 2. A {@link ParDo} splits the resulting query into {@code numQuerySplits} and
|
||||
* assign each split query a unique {@code Integer} as the key. The resulting output is
|
||||
* of the type {@code PCollection<KV<Integer, Query>>}.
|
||||
*
|
||||
* If the value of {@code numQuerySplits} is less than or equal to 0, then the number of
|
||||
* splits will be computed dynamically based on the size of the data for the {@code query}.
|
||||
*
|
||||
* 3. The resulting {@code PCollection} is sharded using a {@link GroupByKey} operation. The
|
||||
* queries are extracted from they {@code KV<Integer, Iterable<Query>>} and flattened to
|
||||
* output a {@code PCollection<Query>}.
|
||||
*
|
||||
* 4. In the third step, a {@code ParDo} reads entities for each query and outputs
|
||||
* a {@code PCollection<Entity>}.
|
||||
*/
|
||||
|
||||
PCollection<Query> inputQuery =
|
||||
gqlQueries.apply(ParDo.of(new GqlQueryTranslateFn(v1Options)));
|
||||
|
||||
return inputQuery
|
||||
.apply("Split", ParDo.of(new SplitQueryFn(v1Options, getNumQuerySplits())))
|
||||
.apply("Reshuffle", Reshuffle.viaRandomKey())
|
||||
.apply("Read", ParDo.of(new ReadFn(v1Options)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void populateDisplayData(DisplayData.Builder builder) {
|
||||
super.populateDisplayData(builder);
|
||||
builder
|
||||
.addIfNotNull(DisplayData.item("projectId", getProjectId()).withLabel("ProjectId"))
|
||||
.addIfNotNull(DisplayData.item("namespace", getNamespace()).withLabel("Namespace"));
|
||||
}
|
||||
|
||||
private static class V1Options implements HasDisplayData, Serializable {
|
||||
private final String project;
|
||||
private final @Nullable String namespace;
|
||||
private final @Nullable String localhost;
|
||||
|
||||
private V1Options(String project, @Nullable String namespace, @Nullable String localhost) {
|
||||
this.project = project;
|
||||
this.namespace = namespace;
|
||||
this.localhost = localhost;
|
||||
}
|
||||
|
||||
public static V1Options from(
|
||||
String projectId, @Nullable String namespace, @Nullable String localhost) {
|
||||
return new V1Options(projectId, namespace, localhost);
|
||||
}
|
||||
|
||||
public String getProjectId() {
|
||||
return project;
|
||||
}
|
||||
|
||||
public @Nullable String getNamespace() {
|
||||
return namespace;
|
||||
}
|
||||
|
||||
public @Nullable String getLocalhost() {
|
||||
return localhost;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void populateDisplayData(DisplayData.Builder builder) {
|
||||
builder
|
||||
.addIfNotNull(DisplayData.item("projectId", getProjectId()).withLabel("ProjectId"))
|
||||
.addIfNotNull(DisplayData.item("namespace", getNamespace()).withLabel("Namespace"));
|
||||
}
|
||||
}
|
||||
|
||||
/** A DoFn that translates a Cloud Datastore gql query string to {@code Query}. */
|
||||
static class GqlQueryTranslateFn extends DoFn<String, Query> {
|
||||
private final V1Options v1Options;
|
||||
private transient Datastore datastore;
|
||||
private final V1DatastoreFactory datastoreFactory;
|
||||
|
||||
GqlQueryTranslateFn(V1Options options) {
|
||||
this(options, new V1DatastoreFactory());
|
||||
}
|
||||
|
||||
GqlQueryTranslateFn(V1Options options, V1DatastoreFactory datastoreFactory) {
|
||||
this.v1Options = options;
|
||||
this.datastoreFactory = datastoreFactory;
|
||||
}
|
||||
|
||||
@StartBundle
|
||||
public void startBundle(StartBundleContext c) {
|
||||
datastore =
|
||||
datastoreFactory.getDatastore(
|
||||
c.getPipelineOptions(), v1Options.getProjectId(), v1Options.getLocalhost());
|
||||
}
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(ProcessContext c) throws Exception {
|
||||
String gqlQuery = c.element();
|
||||
logger.atInfo().log("User query: '%s'.", gqlQuery);
|
||||
Query query =
|
||||
translateGqlQueryWithLimitCheck(gqlQuery, datastore, v1Options.getNamespace());
|
||||
logger.atInfo().log("User gql query translated to Query(%s).", query);
|
||||
c.output(query);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A {@link DoFn} that splits a given query into multiple sub-queries, assigns them unique keys
|
||||
* and outputs them as {@link KV}.
|
||||
*/
|
||||
private static class SplitQueryFn extends DoFn<Query, Query> {
|
||||
private final V1Options options;
|
||||
// number of splits to make for a given query
|
||||
private final int numSplits;
|
||||
|
||||
private final V1DatastoreFactory datastoreFactory;
|
||||
// Datastore client
|
||||
private transient Datastore datastore;
|
||||
// Query splitter
|
||||
private transient QuerySplitter querySplitter;
|
||||
|
||||
public SplitQueryFn(V1Options options, int numSplits) {
|
||||
this(options, numSplits, new V1DatastoreFactory());
|
||||
}
|
||||
|
||||
private SplitQueryFn(V1Options options, int numSplits, V1DatastoreFactory datastoreFactory) {
|
||||
this.options = options;
|
||||
this.numSplits = numSplits;
|
||||
this.datastoreFactory = datastoreFactory;
|
||||
}
|
||||
|
||||
@StartBundle
|
||||
public void startBundle(StartBundleContext c) {
|
||||
datastore =
|
||||
datastoreFactory.getDatastore(
|
||||
c.getPipelineOptions(), options.getProjectId(), options.getLocalhost());
|
||||
querySplitter = datastoreFactory.getQuerySplitter();
|
||||
}
|
||||
|
||||
@ProcessElement
|
||||
public void processElement(ProcessContext c) {
|
||||
Query query = c.element();
|
||||
|
||||
// If query has a user set limit, then do not split.
|
||||
if (query.hasLimit()) {
|
||||
c.output(query);
|
||||
return;
|
||||
}
|
||||
|
||||
int estimatedNumSplits;
|
||||
// Compute the estimated numSplits if numSplits is not specified by the user.
|
||||
if (numSplits <= 0) {
|
||||
estimatedNumSplits = getEstimatedNumSplits(datastore, query, options.getNamespace());
|
||||
} else {
|
||||
estimatedNumSplits = numSplits;
|
||||
}
|
||||
|
||||
logger.atInfo().log("Splitting the query into %d splits.", estimatedNumSplits);
|
||||
List<Query> querySplits;
|
||||
try {
|
||||
querySplits =
|
||||
splitQuery(
|
||||
query, options.getNamespace(), datastore, querySplitter, estimatedNumSplits);
|
||||
} catch (Exception e) {
|
||||
logger.atWarning().log("Unable to parallelize the given query: %s", query, e);
|
||||
querySplits = ImmutableList.of(query);
|
||||
}
|
||||
|
||||
// assign unique keys to query splits.
|
||||
for (Query subquery : querySplits) {
|
||||
c.output(subquery);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void populateDisplayData(DisplayData.Builder builder) {
|
||||
super.populateDisplayData(builder);
|
||||
builder.include("options", options);
|
||||
if (numSplits > 0) {
|
||||
builder.add(
|
||||
DisplayData.item("numQuerySplits", numSplits)
|
||||
.withLabel("Requested number of Query splits"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A {@link DoFn} that reads entities from Cloud Datastore for each query. */
|
||||
private static class ReadFn extends DoFn<Query, Entity> {
|
||||
private final V1Options options;
|
||||
private final V1DatastoreFactory datastoreFactory;
|
||||
// Datastore client
|
||||
private transient Datastore datastore;
|
||||
private final Counter rpcErrors = Metrics.counter(ReadFn.class, "datastoreRpcErrors");
|
||||
private final Counter rpcSuccesses = Metrics.counter(ReadFn.class, "datastoreRpcSuccesses");
|
||||
private static final int MAX_RETRIES = 5;
|
||||
private static final FluentBackoff RUNQUERY_BACKOFF =
|
||||
FluentBackoff.DEFAULT
|
||||
.withMaxRetries(MAX_RETRIES)
|
||||
.withInitialBackoff(Duration.standardSeconds(5));
|
||||
|
||||
public ReadFn(V1Options options) {
|
||||
this(options, new V1DatastoreFactory());
|
||||
}
|
||||
|
||||
private ReadFn(V1Options options, V1DatastoreFactory datastoreFactory) {
|
||||
this.options = options;
|
||||
this.datastoreFactory = datastoreFactory;
|
||||
}
|
||||
|
||||
@StartBundle
|
||||
public void startBundle(StartBundleContext c) {
|
||||
datastore =
|
||||
datastoreFactory.getDatastore(
|
||||
c.getPipelineOptions(), options.getProjectId(), options.getLocalhost());
|
||||
}
|
||||
|
||||
private RunQueryResponse runQueryWithRetries(RunQueryRequest request) throws Exception {
|
||||
Sleeper sleeper = Sleeper.DEFAULT;
|
||||
BackOff backoff = RUNQUERY_BACKOFF.backoff();
|
||||
while (true) {
|
||||
try {
|
||||
RunQueryResponse response = datastore.runQuery(request);
|
||||
rpcSuccesses.inc();
|
||||
return response;
|
||||
} catch (DatastoreException exception) {
|
||||
rpcErrors.inc();
|
||||
|
||||
if (NON_RETRYABLE_ERRORS.contains(exception.getCode())) {
|
||||
throw exception;
|
||||
}
|
||||
if (!BackOffUtils.next(sleeper, backoff)) {
|
||||
logger.atSevere().log("Aborting after %d retries.", MAX_RETRIES);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Read and output entities for the given query. */
|
||||
@ProcessElement
|
||||
public void processElement(ProcessContext context) throws Exception {
|
||||
Query query = context.element();
|
||||
String namespace = options.getNamespace();
|
||||
int userLimit = query.hasLimit() ? query.getLimit().getValue() : Integer.MAX_VALUE;
|
||||
|
||||
boolean moreResults = true;
|
||||
QueryResultBatch currentBatch = null;
|
||||
|
||||
while (moreResults) {
|
||||
Query.Builder queryBuilder = query.toBuilder();
|
||||
queryBuilder.setLimit(
|
||||
Int32Value.newBuilder().setValue(Math.min(userLimit, QUERY_BATCH_LIMIT)));
|
||||
|
||||
if (currentBatch != null && !currentBatch.getEndCursor().isEmpty()) {
|
||||
queryBuilder.setStartCursor(currentBatch.getEndCursor());
|
||||
}
|
||||
|
||||
RunQueryRequest request = makeRequest(queryBuilder.build(), namespace);
|
||||
RunQueryResponse response = runQueryWithRetries(request);
|
||||
|
||||
currentBatch = response.getBatch();
|
||||
|
||||
// MORE_RESULTS_AFTER_LIMIT is not implemented yet:
|
||||
// https://groups.google.com/forum/#!topic/gcd-discuss/iNs6M1jA2Vw, so
|
||||
// use result count to determine if more results might exist.
|
||||
int numFetch = currentBatch.getEntityResultsCount();
|
||||
if (query.hasLimit()) {
|
||||
verify(
|
||||
userLimit >= numFetch,
|
||||
"Expected userLimit %s >= numFetch %s, because query limit %s must be <= userLimit",
|
||||
userLimit,
|
||||
numFetch,
|
||||
query.getLimit());
|
||||
userLimit -= numFetch;
|
||||
}
|
||||
|
||||
// output all the entities from the current batch.
|
||||
for (EntityResult entityResult : currentBatch.getEntityResultsList()) {
|
||||
context.output(entityResult.getEntity());
|
||||
}
|
||||
|
||||
// Check if we have more entities to be read.
|
||||
moreResults =
|
||||
// User-limit does not exist (so userLimit == MAX_VALUE) and/or has not been satisfied
|
||||
(userLimit > 0)
|
||||
// All indications from the API are that there are/may be more results.
|
||||
&& ((numFetch == QUERY_BATCH_LIMIT)
|
||||
|| (currentBatch.getMoreResults() == NOT_FINISHED));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void populateDisplayData(DisplayData.Builder builder) {
|
||||
super.populateDisplayData(builder);
|
||||
builder.include("options", options);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A wrapper factory class for Cloud Datastore singleton classes {@link DatastoreFactory} and
|
||||
* {@link QuerySplitter}
|
||||
*
|
||||
* <p>{@link DatastoreFactory} and {@link QuerySplitter} are not java serializable, hence wrapping
|
||||
* them under this class, which implements {@link Serializable}.
|
||||
*/
|
||||
private static class V1DatastoreFactory implements Serializable {
|
||||
|
||||
/** Builds a Cloud Datastore client for the given pipeline options and project. */
|
||||
public Datastore getDatastore(PipelineOptions pipelineOptions, String projectId) {
|
||||
return getDatastore(pipelineOptions, projectId, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a Cloud Datastore client for the given pipeline options, project and an optional
|
||||
* locahost.
|
||||
*/
|
||||
public Datastore getDatastore(
|
||||
PipelineOptions pipelineOptions, String projectId, @Nullable String localhost) {
|
||||
Credentials credential = pipelineOptions.as(GcpOptions.class).getGcpCredential();
|
||||
HttpRequestInitializer initializer;
|
||||
if (credential != null) {
|
||||
initializer =
|
||||
new ChainingHttpRequestInitializer(
|
||||
new HttpCredentialsAdapter(credential), new RetryHttpRequestInitializer());
|
||||
} else {
|
||||
initializer = new RetryHttpRequestInitializer();
|
||||
}
|
||||
|
||||
DatastoreOptions.Builder builder =
|
||||
new DatastoreOptions.Builder().projectId(projectId).initializer(initializer);
|
||||
|
||||
if (localhost != null) {
|
||||
builder.localHost(localhost);
|
||||
} else {
|
||||
builder.host("batch-datastore.googleapis.com");
|
||||
}
|
||||
|
||||
return DatastoreFactory.get().create(builder.build());
|
||||
}
|
||||
|
||||
/** Builds a Cloud Datastore {@link QuerySplitter}. */
|
||||
public QuerySplitter getQuerySplitter() {
|
||||
return DatastoreHelper.getQuerySplitter();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.rde.RdeModule.BRDA_QUEUE;
|
||||
import static google.registry.rde.RdeModule.RDE_UPLOAD_QUEUE;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
@@ -249,9 +248,6 @@ public class RdeIO {
|
||||
// Now that we're done, output roll the cursor forward.
|
||||
if (key.manual()) {
|
||||
logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task.");
|
||||
// Temporary measure to run RDE in beam in parallel with the daily MapReduce based RDE runs.
|
||||
} else if (tm().isOfy()) {
|
||||
logger.atInfo().log("Ofy is primary TM; not advancing cursor or enqueuing upload task.");
|
||||
} else {
|
||||
outputReceiver.output(KV.of(key, revision));
|
||||
}
|
||||
@@ -273,16 +269,15 @@ public class RdeIO {
|
||||
@ProcessElement
|
||||
public void processElement(
|
||||
@Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
|
||||
|
||||
tm().transact(
|
||||
() -> {
|
||||
PendingDeposit key = input.getKey();
|
||||
Registry registry = Registry.get(key.tld());
|
||||
Optional<Cursor> cursor =
|
||||
transactIfJpaTm(
|
||||
() ->
|
||||
tm().loadByKeyIfPresent(
|
||||
Cursor.createVKey(key.cursor(), registry.getTldStr())));
|
||||
tm().transact(
|
||||
() ->
|
||||
tm().loadByKeyIfPresent(
|
||||
Cursor.createScopedVKey(key.cursor(), registry)));
|
||||
DateTime position = getCursorTimeOrStartOfTime(cursor);
|
||||
checkState(key.interval() != null, "Interval must be present");
|
||||
DateTime newPosition = key.watermark().plus(key.interval());
|
||||
@@ -295,7 +290,7 @@ public class RdeIO {
|
||||
"Partial ordering of RDE deposits broken: %s %s",
|
||||
position,
|
||||
key);
|
||||
tm().put(Cursor.create(key.cursor(), newPosition, registry));
|
||||
tm().put(Cursor.createScoped(key.cursor(), newPosition, registry));
|
||||
logger.atInfo().log(
|
||||
"Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
|
||||
RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), input.getValue());
|
||||
|
||||
@@ -27,14 +27,12 @@ import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.persistence.transaction.JpaTransactionManager;
|
||||
import google.registry.util.TaskQueueUtils;
|
||||
import google.registry.util.YamlUtils;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.lang.annotation.Retention;
|
||||
@@ -43,7 +41,9 @@ import java.net.URL;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.annotation.Nullable;
|
||||
import javax.inject.Named;
|
||||
@@ -225,7 +225,7 @@ public final class RegistryConfig {
|
||||
/**
|
||||
* Returns the Google Cloud Storage bucket for storing zone files.
|
||||
*
|
||||
* @see google.registry.backup.ExportCommitLogDiffAction
|
||||
* @see google.registry.tools.server.GenerateZoneFilesAction
|
||||
*/
|
||||
@Provides
|
||||
@Config("zoneFilesBucket")
|
||||
@@ -233,22 +233,11 @@ public final class RegistryConfig {
|
||||
return projectId + "-zonefiles";
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Google Cloud Storage bucket for storing commit logs.
|
||||
*
|
||||
* @see google.registry.backup.ExportCommitLogDiffAction
|
||||
*/
|
||||
/** @see RegistryConfig#getDatabaseRetention() */
|
||||
@Provides
|
||||
@Config("commitLogGcsBucket")
|
||||
public static String provideCommitLogGcsBucket(@Config("projectId") String projectId) {
|
||||
return projectId + "-commits";
|
||||
}
|
||||
|
||||
/** @see RegistryConfig#getCommitLogDatastoreRetention() */
|
||||
@Provides
|
||||
@Config("commitLogDatastoreRetention")
|
||||
public static Duration provideCommitLogDatastoreRetention() {
|
||||
return RegistryConfig.getCommitLogDatastoreRetention();
|
||||
@Config("databaseRetention")
|
||||
public static Duration provideDatabaseRetention() {
|
||||
return RegistryConfig.getDatabaseRetention();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -262,18 +251,6 @@ public final class RegistryConfig {
|
||||
return projectId + "-domain-lists";
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch size for the number of transactions' worth of commit log data to process at once when
|
||||
* exporting a commit log diff.
|
||||
*
|
||||
* @see google.registry.backup.ExportCommitLogDiffAction
|
||||
*/
|
||||
@Provides
|
||||
@Config("commitLogDiffExportBatchSize")
|
||||
public static int provideCommitLogDiffExportBatchSize() {
|
||||
return 100;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Google Cloud Storage bucket for staging BRDA escrow deposits.
|
||||
*
|
||||
@@ -765,17 +742,6 @@ public final class RegistryConfig {
|
||||
return config.rde.reportUrlPrefix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maximum amount of time generating an escrow deposit for a TLD could take, before killing.
|
||||
*
|
||||
* @see google.registry.rde.RdeStagingReducer
|
||||
*/
|
||||
@Provides
|
||||
@Config("rdeStagingLockTimeout")
|
||||
public static Duration provideRdeStagingLockTimeout() {
|
||||
return Duration.standardHours(2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Maximum amount of time it should ever take to upload an escrow deposit, before killing.
|
||||
*
|
||||
@@ -792,7 +758,7 @@ public final class RegistryConfig {
|
||||
*
|
||||
* <p>This value was communicated to us by the escrow provider.
|
||||
*
|
||||
* @see google.registry.rde.RdeStagingReducer
|
||||
* @see google.registry.rde.RdeUploadAction
|
||||
*/
|
||||
@Provides
|
||||
@Config("rdeUploadSftpCooldown")
|
||||
@@ -952,7 +918,7 @@ public final class RegistryConfig {
|
||||
* <p>Note that this uses {@code @Named} instead of {@code @Config} so that it can be used from
|
||||
* the low-level util package, which cannot have a dependency on the config package.
|
||||
*
|
||||
* @see TaskQueueUtils
|
||||
* @see google.registry.util.CloudTasksUtils
|
||||
*/
|
||||
@Provides
|
||||
@Named("transientFailureRetries")
|
||||
@@ -1088,8 +1054,8 @@ public final class RegistryConfig {
|
||||
* @see google.registry.batch.AsyncTaskEnqueuer
|
||||
*/
|
||||
@Provides
|
||||
@Config("asyncDeleteFlowMapreduceDelay")
|
||||
public static Duration provideAsyncDeleteFlowMapreduceDelay(RegistryConfigSettings config) {
|
||||
@Config("asyncDeleteDelay")
|
||||
public static Duration provideAsyncDeleteDelay(RegistryConfigSettings config) {
|
||||
return Duration.standardSeconds(config.misc.asyncDeleteDelaySeconds);
|
||||
}
|
||||
|
||||
@@ -1301,7 +1267,7 @@ public final class RegistryConfig {
|
||||
e.getKey().equals("START_OF_TIME")
|
||||
? START_OF_TIME
|
||||
: DateTime.parse(e.getKey()),
|
||||
e -> e.getValue()));
|
||||
Entry::getValue));
|
||||
}
|
||||
|
||||
@Provides
|
||||
@@ -1365,41 +1331,11 @@ public final class RegistryConfig {
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Google Cloud Storage bucket for storing Datastore backups.
|
||||
* Returns the length of time before commit logs should be deleted from the database.
|
||||
*
|
||||
* @see google.registry.export.BackupDatastoreAction
|
||||
* @see google.registry.tools.server.GenerateZoneFilesAction
|
||||
*/
|
||||
public static String getDatastoreBackupsBucket() {
|
||||
return "gs://" + getProjectId() + "-datastore-backups";
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of sharded commit log buckets.
|
||||
*
|
||||
* <p>This number is crucial for determining how much transactional throughput the system can
|
||||
* allow, because it determines how many entity groups are available for writing commit logs.
|
||||
* Since entity groups have a one transaction per second SLA (which is actually like ten in
|
||||
* practice), a registry that wants to be able to handle one hundred transactions per second
|
||||
* should have one hundred buckets.
|
||||
*
|
||||
* <p><b>Warning:</b> This can be raised but never lowered.
|
||||
*
|
||||
* @see google.registry.model.ofy.CommitLogBucket
|
||||
*/
|
||||
public static int getCommitLogBucketCount() {
|
||||
return CONFIG_SETTINGS.get().datastore.commitLogBucketsNum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the length of time before commit logs should be deleted from Datastore.
|
||||
*
|
||||
* <p>The only reason you'll want to retain this commit logs in Datastore is for performing
|
||||
* point-in-time restoration queries for subsystems like RDE.
|
||||
*
|
||||
* @see google.registry.backup.DeleteOldCommitLogsAction
|
||||
* @see google.registry.model.translators.CommitLogRevisionsTranslatorFactory
|
||||
*/
|
||||
public static Duration getCommitLogDatastoreRetention() {
|
||||
public static Duration getDatabaseRetention() {
|
||||
return Duration.standardDays(30);
|
||||
}
|
||||
|
||||
|
||||
@@ -107,7 +107,6 @@ public class RegistryConfigSettings {
|
||||
|
||||
/** Configuration for Cloud Datastore. */
|
||||
public static class Datastore {
|
||||
public int commitLogBucketsNum;
|
||||
public int eppResourceIndexBucketsNum;
|
||||
public int baseOfyRetryMillis;
|
||||
}
|
||||
|
||||
@@ -183,10 +183,6 @@ registryPolicy:
|
||||
requireSslCertificates: true
|
||||
|
||||
datastore:
|
||||
# Number of commit log buckets in Datastore. Lowering this after initial
|
||||
# install risks losing up to a days' worth of differential backups.
|
||||
commitLogBucketsNum: 397
|
||||
|
||||
# Number of EPP resource index buckets in Datastore. Don’t change after
|
||||
# initial install.
|
||||
eppResourceIndexBucketsNum: 997
|
||||
|
||||
@@ -11,7 +11,6 @@ registryPolicy:
|
||||
Line 2 is this 1.
|
||||
|
||||
datastore:
|
||||
commitLogBucketsNum: 3
|
||||
eppResourceIndexBucketsNum: 3
|
||||
baseOfyRetryMillis: 0
|
||||
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.cron;
|
||||
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Service;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Action for fanning out cron tasks for each commit log bucket. */
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/cron/commitLogFanout",
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
public final class CommitLogFanoutAction implements Runnable {
|
||||
|
||||
public static final String BUCKET_PARAM = "bucket";
|
||||
|
||||
@Inject CloudTasksUtils cloudTasksUtils;
|
||||
|
||||
@Inject @Parameter("endpoint") String endpoint;
|
||||
@Inject @Parameter("queue") String queue;
|
||||
@Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
|
||||
@Inject CommitLogFanoutAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
for (int bucketId : CommitLogBucket.getBucketIds()) {
|
||||
cloudTasksUtils.enqueue(
|
||||
queue,
|
||||
cloudTasksUtils.createPostTaskWithJitter(
|
||||
endpoint,
|
||||
Service.BACKEND.toString(),
|
||||
ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
|
||||
jitterSeconds));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,6 @@ import google.registry.dns.DnsConstants.TargetType;
|
||||
import google.registry.model.tld.Registries;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import google.registry.util.SystemClock;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.logging.Level;
|
||||
@@ -83,17 +82,6 @@ public class DnsQueue {
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a new instance.
|
||||
*
|
||||
* <p><b>Note:</b> Prefer <code>@Inject</code>ing DnsQueue instances instead. You should only use
|
||||
* this helper method in situations for which injection does not work, e.g. inside mapper or
|
||||
* reducer classes in mapreduces that need to be Serializable.
|
||||
*/
|
||||
public static DnsQueue create() {
|
||||
return new DnsQueue(getQueue(DNS_PULL_QUEUE_NAME), new SystemClock());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static DnsQueue createForTesting(Clock clock) {
|
||||
return new DnsQueue(getQueue(DNS_PULL_QUEUE_NAME), clock);
|
||||
|
||||
@@ -30,14 +30,13 @@ import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_CREATED;
|
||||
import static google.registry.request.RequestParameters.PARAM_TLD;
|
||||
import static google.registry.util.DomainNameUtils.getSecondLevelDomain;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.cloud.tasks.v2.Task;
|
||||
import com.google.common.collect.ComparisonChain;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.Iterables;
|
||||
@@ -50,20 +49,19 @@ import google.registry.dns.DnsConstants.TargetType;
|
||||
import google.registry.model.tld.Registries;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Service;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.TaskQueueUtils;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Random;
|
||||
import java.util.stream.Collectors;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
@@ -84,7 +82,6 @@ import org.joda.time.Duration;
|
||||
public final class ReadDnsQueueAction implements Runnable {
|
||||
|
||||
private static final String PARAM_JITTER_SECONDS = "jitterSeconds";
|
||||
private static final Random random = new Random();
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
/**
|
||||
@@ -101,15 +98,31 @@ public final class ReadDnsQueueAction implements Runnable {
|
||||
*/
|
||||
private static final Duration LEASE_PADDING = Duration.standardMinutes(1);
|
||||
|
||||
@Inject @Config("dnsTldUpdateBatchSize") int tldUpdateBatchSize;
|
||||
@Inject @Config("readDnsQueueActionRuntime") Duration requestedMaximumDuration;
|
||||
@Inject @Named(DNS_PUBLISH_PUSH_QUEUE_NAME) Queue dnsPublishPushQueue;
|
||||
@Inject @Parameter(PARAM_JITTER_SECONDS) Optional<Integer> jitterSeconds;
|
||||
@Inject Clock clock;
|
||||
@Inject DnsQueue dnsQueue;
|
||||
@Inject HashFunction hashFunction;
|
||||
@Inject TaskQueueUtils taskQueueUtils;
|
||||
@Inject ReadDnsQueueAction() {}
|
||||
private final int tldUpdateBatchSize;
|
||||
private final Duration requestedMaximumDuration;
|
||||
private final Optional<Integer> jitterSeconds;
|
||||
private final Clock clock;
|
||||
private final DnsQueue dnsQueue;
|
||||
private final HashFunction hashFunction;
|
||||
private final CloudTasksUtils cloudTasksUtils;
|
||||
|
||||
@Inject
|
||||
ReadDnsQueueAction(
|
||||
@Config("dnsTldUpdateBatchSize") int tldUpdateBatchSize,
|
||||
@Config("readDnsQueueActionRuntime") Duration requestedMaximumDuration,
|
||||
@Parameter(PARAM_JITTER_SECONDS) Optional<Integer> jitterSeconds,
|
||||
Clock clock,
|
||||
DnsQueue dnsQueue,
|
||||
HashFunction hashFunction,
|
||||
CloudTasksUtils cloudTasksUtils) {
|
||||
this.tldUpdateBatchSize = tldUpdateBatchSize;
|
||||
this.requestedMaximumDuration = requestedMaximumDuration;
|
||||
this.jitterSeconds = jitterSeconds;
|
||||
this.clock = clock;
|
||||
this.dnsQueue = dnsQueue;
|
||||
this.hashFunction = hashFunction;
|
||||
this.cloudTasksUtils = cloudTasksUtils;
|
||||
}
|
||||
|
||||
/** Container for items we pull out of the DNS pull queue and process for fanout. */
|
||||
@AutoValue
|
||||
@@ -322,17 +335,13 @@ public final class ReadDnsQueueAction implements Runnable {
|
||||
if (numPublishLocks <= 1) {
|
||||
enqueueUpdates(tld, 1, 1, tldRefreshItemsEntry.getValue());
|
||||
} else {
|
||||
tldRefreshItemsEntry
|
||||
.getValue()
|
||||
.stream()
|
||||
tldRefreshItemsEntry.getValue().stream()
|
||||
.collect(
|
||||
toImmutableSetMultimap(
|
||||
refreshItem -> getLockIndex(tld, numPublishLocks, refreshItem),
|
||||
refreshItem -> refreshItem))
|
||||
.asMap()
|
||||
.entrySet()
|
||||
.forEach(
|
||||
entry -> enqueueUpdates(tld, entry.getKey(), numPublishLocks, entry.getValue()));
|
||||
.forEach((key, value) -> enqueueUpdates(tld, key, numPublishLocks, value));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -340,10 +349,10 @@ public final class ReadDnsQueueAction implements Runnable {
|
||||
/**
|
||||
* Returns the lock index for a given refreshItem.
|
||||
*
|
||||
* <p>We hash the second level domain domain for all records, to group in-balliwick hosts (the
|
||||
* only ones we refresh DNS for) with their superordinate domains. We use consistent hashing to
|
||||
* determine the lock index because it gives us [0,N) bucketing properties out of the box, then
|
||||
* add 1 to make indexes within [1,N].
|
||||
* <p>We hash the second level domain for all records, to group in-bailiwick hosts (the only ones
|
||||
* we refresh DNS for) with their superordinate domains. We use consistent hashing to determine
|
||||
* the lock index because it gives us [0,N) bucketing properties out of the box, then add 1 to
|
||||
* make indexes within [1,N].
|
||||
*/
|
||||
private int getLockIndex(String tld, int numPublishLocks, RefreshItem refreshItem) {
|
||||
String domain = getSecondLevelDomain(refreshItem.name(), tld);
|
||||
@@ -360,33 +369,32 @@ public final class ReadDnsQueueAction implements Runnable {
|
||||
DateTime earliestCreateTime =
|
||||
chunk.stream().map(RefreshItem::creationTime).min(Comparator.naturalOrder()).get();
|
||||
for (String dnsWriter : Registry.get(tld).getDnsWriters()) {
|
||||
taskQueueUtils.enqueue(
|
||||
dnsPublishPushQueue,
|
||||
TaskOptions.Builder.withUrl(PublishDnsUpdatesAction.PATH)
|
||||
.countdownMillis(
|
||||
jitterSeconds
|
||||
.map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
|
||||
.orElse(0))
|
||||
.param(PARAM_TLD, tld)
|
||||
.param(PARAM_DNS_WRITER, dnsWriter)
|
||||
.param(PARAM_LOCK_INDEX, Integer.toString(lockIndex))
|
||||
.param(PARAM_NUM_PUBLISH_LOCKS, Integer.toString(numPublishLocks))
|
||||
.param(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
|
||||
.param(PARAM_REFRESH_REQUEST_CREATED, earliestCreateTime.toString())
|
||||
.param(
|
||||
PARAM_DOMAINS,
|
||||
chunk
|
||||
.stream()
|
||||
.filter(item -> item.type() == TargetType.DOMAIN)
|
||||
.map(RefreshItem::name)
|
||||
.collect(Collectors.joining(",")))
|
||||
.param(
|
||||
PARAM_HOSTS,
|
||||
chunk
|
||||
.stream()
|
||||
.filter(item -> item.type() == TargetType.HOST)
|
||||
.map(RefreshItem::name)
|
||||
.collect(Collectors.joining(","))));
|
||||
Task task =
|
||||
cloudTasksUtils.createPostTaskWithJitter(
|
||||
PublishDnsUpdatesAction.PATH,
|
||||
Service.BACKEND.toString(),
|
||||
ImmutableMultimap.<String, String>builder()
|
||||
.put(PARAM_TLD, tld)
|
||||
.put(PARAM_DNS_WRITER, dnsWriter)
|
||||
.put(PARAM_LOCK_INDEX, Integer.toString(lockIndex))
|
||||
.put(PARAM_NUM_PUBLISH_LOCKS, Integer.toString(numPublishLocks))
|
||||
.put(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
|
||||
.put(PARAM_REFRESH_REQUEST_CREATED, earliestCreateTime.toString())
|
||||
.put(
|
||||
PARAM_DOMAINS,
|
||||
chunk.stream()
|
||||
.filter(item -> item.type() == TargetType.DOMAIN)
|
||||
.map(RefreshItem::name)
|
||||
.collect(Collectors.joining(",")))
|
||||
.put(
|
||||
PARAM_HOSTS,
|
||||
chunk.stream()
|
||||
.filter(item -> item.type() == TargetType.HOST)
|
||||
.map(RefreshItem::name)
|
||||
.collect(Collectors.joining(",")))
|
||||
.build(),
|
||||
jitterSeconds);
|
||||
cloudTasksUtils.enqueue(DNS_PUBLISH_PUSH_QUEUE_NAME, task);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +83,6 @@
|
||||
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
|
||||
<description>
|
||||
This job resaves all our resources, projected in time to "now".
|
||||
It is needed for "deleteOldCommitLogs" to work correctly.
|
||||
</description>
|
||||
<schedule>1st monday of month 09:00</schedule>
|
||||
<target>backend</target>
|
||||
@@ -92,9 +91,9 @@
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url>
|
||||
<description>
|
||||
This job runs a mapreduce that creates synthetic OneTime billing events from Recurring billing
|
||||
This job runs an action that creates synthetic OneTime billing events from Recurring billing
|
||||
events. Events are created for all instances of Recurring billing events that should exist
|
||||
between the RECURRING_BILLING cursor's time and the execution time of the mapreduce.
|
||||
between the RECURRING_BILLING cursor's time and the execution time of the action.
|
||||
</description>
|
||||
<schedule>every day 03:00</schedule>
|
||||
<target>backend</target>
|
||||
|
||||
@@ -169,84 +169,12 @@
|
||||
<url-pattern>/_dr/dnsRefresh</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Exports a Datastore backup snapshot to GCS. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/backupDatastore</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Checks the completion of a Datastore backup snapshot. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/checkDatastoreBackup</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Loads a Datastore backup snapshot into BigQuery. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/uploadDatastoreBackup</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Updates a view to point at a certain snapshot in BigQuery. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/updateSnapshotView</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Polls state of jobs in Bigquery -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/pollBigqueryJob</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Fans out a cron task over an adjustable range of TLDs. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/cron/fanout</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Backups. -->
|
||||
|
||||
<!-- Fans out a cron task over all commit log buckets. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/cron/commitLogFanout</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Deletes old commit logs from Datastore. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/deleteOldCommitLogs</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Checkpoints commit logs. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/cron/commitLogCheckpoint</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Exports commit log diffs. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/exportCommitLogDiff</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Deletes commit logs. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/killCommitLogs</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- MapReduce servlet. -->
|
||||
<servlet>
|
||||
<servlet-name>mapreduce</servlet-name>
|
||||
<servlet-class>com.google.appengine.tools.mapreduce.MapReduceServlet</servlet-class>
|
||||
</servlet>
|
||||
<servlet-mapping>
|
||||
<servlet-name>mapreduce</servlet-name>
|
||||
<url-pattern>/_dr/mapreduce/*</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Pipeline GUI servlets. -->
|
||||
<servlet>
|
||||
<servlet-name>pipeline</servlet-name>
|
||||
@@ -286,24 +214,18 @@
|
||||
<url-pattern>/_dr/task/exportDomainLists</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to delete all prober data. -->
|
||||
<!-- Action to delete all prober data. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/deleteProberData</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to delete load test data. -->
|
||||
<!-- Action to delete load test data. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/deleteLoadTestData</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to re-save all EppResources. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/resaveAllEppResources</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Dataflow pipeline to re-save all EPP resources. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
@@ -334,7 +256,7 @@
|
||||
<url-pattern>/_dr/task/refreshDnsOnHostRename</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to expand recurring billing events into OneTimes. -->
|
||||
<!-- Action to expand recurring billing events into OneTimes. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/expandRecurringBillingEvents</url-pattern>
|
||||
@@ -352,30 +274,6 @@
|
||||
<url-pattern>/_dr/task/sendExpiringCertificateNotificationEmail</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to import contacts from escrow file -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/importRdeContacts</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to import hosts from escrow file -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/importRdeHosts</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to import domains from escrow file -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/importRdeDomains</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to link hosts from escrow file to superordinate domains -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/linkRdeHosts</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Action to automatically re-lock a domain after unlocking it -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
@@ -395,12 +293,6 @@ have been in the database for a certain period of time. -->
|
||||
<url-pattern>/_dr/task/wipeOutCloudSql</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Action to wipeout Cloud Datastore data -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>backend-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/wipeOutDatastore</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Security config -->
|
||||
<security-constraint>
|
||||
<web-resource-collection>
|
||||
|
||||
@@ -66,40 +66,6 @@
|
||||
<url-pattern>/_dr/epptool</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to re-save all HistoryEntries. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>tools-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/resaveAllHistoryEntries</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to delete EppResources, children, and indices. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>tools-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/killAllEppResources</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Mapreduce to delete all commit logs. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>tools-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/killAllCommitLogs</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Restores commit logs. -->
|
||||
<servlet-mapping>
|
||||
<servlet-name>tools-servlet</servlet-name>
|
||||
<url-pattern>/_dr/task/restoreCommitLogs</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- This path serves up the App Engine results page for mapreduce runs. -->
|
||||
<servlet>
|
||||
<servlet-name>mapreduce</servlet-name>
|
||||
<servlet-class>com.google.appengine.tools.mapreduce.MapReduceServlet</servlet-class>
|
||||
</servlet>
|
||||
<servlet-mapping>
|
||||
<servlet-name>mapreduce</servlet-name>
|
||||
<url-pattern>/_dr/mapreduce/*</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!-- Pipeline GUI servlets. -->
|
||||
<servlet>
|
||||
<servlet-name>pipeline</servlet-name>
|
||||
|
||||
@@ -106,7 +106,6 @@
|
||||
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
|
||||
<description>
|
||||
This job resaves all our resources, projected in time to "now".
|
||||
It is needed for "deleteOldCommitLogs" to work correctly.
|
||||
</description>
|
||||
<schedule>1st monday of month 09:00</schedule>
|
||||
<target>backend</target>
|
||||
@@ -133,9 +132,9 @@
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url>
|
||||
<description>
|
||||
This job runs a mapreduce that creates synthetic OneTime billing events from Recurring billing
|
||||
This job runs an action that creates synthetic OneTime billing events from Recurring billing
|
||||
events. Events are created for all instances of Recurring billing events that should exist
|
||||
between the RECURRING_BILLING cursor's time and the execution time of the mapreduce.
|
||||
between the RECURRING_BILLING cursor's time and the execution time of the action.
|
||||
</description>
|
||||
<schedule>every day 03:00</schedule>
|
||||
<target>backend</target>
|
||||
|
||||
@@ -35,7 +35,6 @@
|
||||
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
|
||||
<description>
|
||||
This job resaves all our resources, projected in time to "now".
|
||||
It is needed for "deleteOldCommitLogs" to work correctly.
|
||||
</description>
|
||||
<schedule>1st monday of month 09:00</schedule>
|
||||
<target>backend</target>
|
||||
|
||||
@@ -90,7 +90,6 @@
|
||||
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
|
||||
<description>
|
||||
This job resaves all our resources, projected in time to "now".
|
||||
It is needed for "deleteOldCommitLogs" to work correctly.
|
||||
</description>
|
||||
<schedule>1st monday of month 09:00</schedule>
|
||||
<target>backend</target>
|
||||
@@ -108,9 +107,9 @@
|
||||
<cron>
|
||||
<url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url>
|
||||
<description>
|
||||
This job runs a mapreduce that creates synthetic OneTime billing events from Recurring billing
|
||||
This job runs an action that creates synthetic OneTime billing events from Recurring billing
|
||||
events. Events are created for all instances of Recurring billing events that should exist
|
||||
between the RECURRING_BILLING cursor's time and the execution time of the mapreduce.
|
||||
between the RECURRING_BILLING cursor's time and the execution time of the action.
|
||||
</description>
|
||||
<schedule>every day 03:00</schedule>
|
||||
<target>backend</target>
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
|
||||
import static google.registry.util.TypeUtils.hasAnnotation;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Ordering;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.EntityClasses;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.annotations.InCrossTld;
|
||||
import google.registry.model.annotations.NotBackedUp;
|
||||
import google.registry.model.annotations.ReportedOn;
|
||||
import google.registry.model.annotations.VirtualEntity;
|
||||
|
||||
/** Constants related to export code. */
|
||||
@DeleteAfterMigration
|
||||
public final class AnnotatedEntities {
|
||||
|
||||
/** Returns the names of kinds to include in Datastore backups. */
|
||||
public static ImmutableSet<String> getBackupKinds() {
|
||||
// Back up all entity classes that aren't annotated with @VirtualEntity (never even persisted
|
||||
// to Datastore, so they can't be backed up) or @NotBackedUp (intentionally omitted).
|
||||
return EntityClasses.ALL_CLASSES
|
||||
.stream()
|
||||
.filter(hasAnnotation(VirtualEntity.class).negate())
|
||||
.filter(hasAnnotation(NotBackedUp.class).negate())
|
||||
.map(Key::getKind)
|
||||
.collect(toImmutableSortedSet(Ordering.natural()));
|
||||
}
|
||||
|
||||
/** Returns the names of kinds to import into reporting tools (e.g. BigQuery). */
|
||||
public static ImmutableSet<String> getReportingKinds() {
|
||||
return EntityClasses.ALL_CLASSES
|
||||
.stream()
|
||||
.filter(hasAnnotation(ReportedOn.class))
|
||||
.filter(hasAnnotation(VirtualEntity.class).negate())
|
||||
.map(Key::getKind)
|
||||
.collect(toImmutableSortedSet(Ordering.natural()));
|
||||
}
|
||||
|
||||
/** Returns the names of kinds that are in the cross-TLD entity group. */
|
||||
public static ImmutableSet<String> getCrossTldKinds() {
|
||||
return EntityClasses.ALL_CLASSES.stream()
|
||||
.filter(hasAnnotation(InCrossTld.class))
|
||||
.filter(hasAnnotation(VirtualEntity.class).negate())
|
||||
.map(Key::getKind)
|
||||
.collect(toImmutableSortedSet(Ordering.natural()));
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.export.datastore.DatastoreAdmin;
|
||||
import google.registry.export.datastore.Operation;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Service;
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage.
|
||||
*
|
||||
* <p>This is the first step of a four step workflow for exporting snapshots, with each step calling
|
||||
* the next upon successful completion:
|
||||
*
|
||||
* <ol>
|
||||
* <li>The snapshot is exported to Google Cloud Storage (this action).
|
||||
* <li>The {@link CheckBackupAction} polls until the export is completed.
|
||||
* <li>The {@link UploadDatastoreBackupAction} uploads the data from GCS to BigQuery.
|
||||
* <li>The {@link UpdateSnapshotViewAction} updates the view in latest_datastore_export.
|
||||
* </ol>
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = BackupDatastoreAction.PATH,
|
||||
method = POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class BackupDatastoreAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
/** Queue to use for enqueuing the task that will actually launch the backup. */
|
||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||
|
||||
static final String PATH = "/_dr/task/backupDatastore"; // See web.xml.
|
||||
|
||||
@Inject DatastoreAdmin datastoreAdmin;
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject CloudTasksUtils cloudTasksUtils;
|
||||
|
||||
@Inject
|
||||
BackupDatastoreAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
Operation backup =
|
||||
datastoreAdmin
|
||||
.export(
|
||||
RegistryConfig.getDatastoreBackupsBucket(), AnnotatedEntities.getBackupKinds())
|
||||
.execute();
|
||||
|
||||
String backupName = backup.getName();
|
||||
// Enqueue a poll task to monitor the backup for completion and load reporting-related kinds
|
||||
// into bigquery.
|
||||
cloudTasksUtils.enqueue(
|
||||
CheckBackupAction.QUEUE,
|
||||
cloudTasksUtils.createPostTaskWithDelay(
|
||||
CheckBackupAction.PATH,
|
||||
Service.BACKEND.toString(),
|
||||
ImmutableMultimap.of(
|
||||
CheckBackupAction.CHECK_BACKUP_NAME_PARAM,
|
||||
backupName,
|
||||
CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM,
|
||||
Joiner.on(',').join(AnnotatedEntities.getReportingKinds())),
|
||||
CheckBackupAction.POLL_COUNTDOWN));
|
||||
String message =
|
||||
String.format(
|
||||
"Datastore backup started with name: %s\nSaving to %s",
|
||||
backupName, backup.getExportFolderUrl());
|
||||
logger.atInfo().log(message);
|
||||
response.setPayload(message);
|
||||
} catch (Throwable e) {
|
||||
throw new InternalServerErrorException("Exception occurred while backing up Datastore", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static google.registry.bigquery.BigqueryUtils.toJobReferenceString;
|
||||
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.model.Job;
|
||||
import com.google.api.services.bigquery.model.JobReference;
|
||||
import com.google.cloud.tasks.v2.Task;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.protobuf.ByteString;
|
||||
import dagger.Lazy;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Header;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.HttpException.NotModifiedException;
|
||||
import google.registry.request.Payload;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/**
|
||||
* An action which polls the state of a bigquery job. If it is completed then it will log its
|
||||
* completion state; otherwise it will return a failure code so that the task will be retried.
|
||||
*
|
||||
* <p>Note that this is AUTH_INTERNAL_ONLY: we don't allow "admin" for this to mitigate a
|
||||
* vulnerability, see b/177308043.
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = BigqueryPollJobAction.PATH,
|
||||
method = {Action.Method.GET, Action.Method.POST},
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_ONLY)
|
||||
public class BigqueryPollJobAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
static final String QUEUE = "export-bigquery-poll"; // See queue.xml
|
||||
static final String PATH = "/_dr/task/pollBigqueryJob"; // See web.xml
|
||||
static final String CHAINED_TASK_QUEUE_HEADER = "X-DomainRegistry-ChainedTaskQueue";
|
||||
static final String PROJECT_ID_HEADER = "X-DomainRegistry-ProjectId";
|
||||
static final String JOB_ID_HEADER = "X-DomainRegistry-JobId";
|
||||
static final Duration POLL_COUNTDOWN = Duration.standardSeconds(20);
|
||||
|
||||
@Inject Bigquery bigquery;
|
||||
@Inject CloudTasksUtils cloudTasksUtils;
|
||||
|
||||
@Inject @Header(CHAINED_TASK_QUEUE_HEADER) Lazy<String> chainedQueueName;
|
||||
@Inject @Header(PROJECT_ID_HEADER) String projectId;
|
||||
@Inject @Header(JOB_ID_HEADER) String jobId;
|
||||
|
||||
@Inject @Payload ByteString payload;
|
||||
|
||||
@Inject BigqueryPollJobAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
boolean jobOutcome =
|
||||
checkJobOutcome(); // Throws a NotModifiedException if the job hasn't completed.
|
||||
// If the job failed, do not enqueue the next step.
|
||||
if (!jobOutcome || payload == null || payload.size() == 0) {
|
||||
return;
|
||||
}
|
||||
// If there is a payload, it's a chained task, so enqueue it.
|
||||
Task task;
|
||||
try {
|
||||
task =
|
||||
(Task)
|
||||
new ObjectInputStream(new ByteArrayInputStream(payload.toByteArray())).readObject();
|
||||
} catch (ClassNotFoundException | IOException e) {
|
||||
throw new BadRequestException("Cannot deserialize task from payload", e);
|
||||
}
|
||||
Task enqueuedTask = cloudTasksUtils.enqueue(chainedQueueName.get(), task);
|
||||
logger.atInfo().log(
|
||||
"Added chained task %s for %s to queue %s: %s",
|
||||
enqueuedTask.getName(),
|
||||
enqueuedTask.getAppEngineHttpRequest().getRelativeUri(),
|
||||
chainedQueueName.get(),
|
||||
enqueuedTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the provided job succeeded, false if it failed, and throws an exception if it
|
||||
* is still pending.
|
||||
*/
|
||||
private boolean checkJobOutcome() {
|
||||
Job job = null;
|
||||
String jobRefString =
|
||||
toJobReferenceString(new JobReference().setProjectId(projectId).setJobId(jobId));
|
||||
|
||||
try {
|
||||
job = bigquery.jobs().get(projectId, jobId).execute();
|
||||
} catch (IOException e) {
|
||||
// We will throw a new exception because done==false, but first log this exception.
|
||||
logger.atWarning().withCause(e).log("Error checking outcome of BigQuery job %s.", jobId);
|
||||
}
|
||||
// If job is not yet done, then throw an exception so that we'll return a failing HTTP status
|
||||
// code and the task will be retried.
|
||||
if (job == null || !job.getStatus().getState().equals("DONE")) {
|
||||
throw new NotModifiedException(jobRefString);
|
||||
}
|
||||
|
||||
// Check if the job ended with an error.
|
||||
if (job.getStatus().getErrorResult() != null) {
|
||||
logger.atSevere().log("Bigquery job failed - %s - %s.", jobRefString, job);
|
||||
return false;
|
||||
}
|
||||
logger.atInfo().log("Bigquery job succeeded - %s.", jobRefString);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -1,195 +0,0 @@
|
||||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.collect.Sets.intersection;
|
||||
import static google.registry.request.Action.Method.GET;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND;
|
||||
|
||||
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.export.datastore.DatastoreAdmin;
|
||||
import google.registry.export.datastore.Operation;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Service;
|
||||
import google.registry.request.HttpException;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
import google.registry.request.HttpException.NoContentException;
|
||||
import google.registry.request.HttpException.NotModifiedException;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.RequestMethod;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.Duration;
|
||||
import org.joda.time.PeriodType;
|
||||
import org.joda.time.format.PeriodFormat;
|
||||
|
||||
/**
|
||||
* Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery.
|
||||
*/
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = CheckBackupAction.PATH,
|
||||
method = {POST, GET},
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class CheckBackupAction implements Runnable {
|
||||
|
||||
/** Parameter names for passing parameters into this action. */
|
||||
static final String CHECK_BACKUP_NAME_PARAM = "name";
|
||||
|
||||
static final String CHECK_BACKUP_KINDS_TO_LOAD_PARAM = "kindsToLoad";
|
||||
|
||||
/** Action-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot-poll"; // See queue.xml.
|
||||
|
||||
static final String PATH = "/_dr/task/checkDatastoreBackup"; // See web.xml.
|
||||
static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2);
|
||||
|
||||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject DatastoreAdmin datastoreAdmin;
|
||||
@Inject Clock clock;
|
||||
@Inject Response response;
|
||||
@Inject CloudTasksUtils cloudTasksUtils;
|
||||
@Inject @RequestMethod Action.Method requestMethod;
|
||||
|
||||
@Inject
|
||||
@Parameter(CHECK_BACKUP_NAME_PARAM)
|
||||
String backupName;
|
||||
|
||||
@Inject
|
||||
@Parameter(CHECK_BACKUP_KINDS_TO_LOAD_PARAM)
|
||||
String kindsToLoadParam;
|
||||
|
||||
@Inject
|
||||
CheckBackupAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
if (requestMethod == POST) {
|
||||
checkAndLoadBackupIfComplete();
|
||||
} else {
|
||||
// This is a GET request.
|
||||
// TODO(weiminyu): consider moving this functionality to Registry tool.
|
||||
response.setPayload(getExportStatus().toPrettyString());
|
||||
}
|
||||
} catch (HttpException e) {
|
||||
// Rethrow and let caller propagate status code and error message to the response.
|
||||
// See google.registry.request.RequestHandler#handleRequest.
|
||||
throw e;
|
||||
} catch (Throwable e) {
|
||||
throw new InternalServerErrorException(
|
||||
"Exception occurred while checking datastore exports.", e);
|
||||
}
|
||||
}
|
||||
|
||||
private Operation getExportStatus() throws IOException {
|
||||
try {
|
||||
return datastoreAdmin.get(backupName).execute();
|
||||
} catch (GoogleJsonResponseException e) {
|
||||
if (e.getStatusCode() == SC_NOT_FOUND) {
|
||||
String message = String.format("Bad backup name %s: %s", backupName, e.getMessage());
|
||||
// TODO(b/19081569): Ideally this would return a 2XX error so the task would not be
|
||||
// retried but we might abandon backups that start late and haven't yet written to
|
||||
// Datastore. We could fix that by replacing this with a two-phase polling strategy.
|
||||
throw new BadRequestException(message, e);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private void checkAndLoadBackupIfComplete() throws IOException {
|
||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
||||
Operation backup = getExportStatus();
|
||||
|
||||
checkArgument(backup.isExport(), "Expecting an export operation: [%s].", backupName);
|
||||
|
||||
if (backup.isProcessing()
|
||||
&& backup.getRunningTime(clock).isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) {
|
||||
// Backup might still be running, so send a 304 to have the task retry.
|
||||
throw new NotModifiedException(
|
||||
String.format(
|
||||
"Datastore backup %s still in progress: %s", backupName, backup.getProgress()));
|
||||
}
|
||||
if (!backup.isSuccessful()) {
|
||||
// Declare the backup a lost cause, and send 204 No Content so the task will
|
||||
// not be retried.
|
||||
String message =
|
||||
String.format(
|
||||
"Datastore backup %s abandoned - not complete after %s. Progress: %s",
|
||||
backupName,
|
||||
PeriodFormat.getDefault()
|
||||
.print(
|
||||
backup
|
||||
.getRunningTime(clock)
|
||||
.toPeriod()
|
||||
.normalizedStandard(PeriodType.dayTime().withMillisRemoved())),
|
||||
backup.getProgress());
|
||||
throw new NoContentException(message);
|
||||
}
|
||||
|
||||
String backupId = backup.getExportId();
|
||||
// Log a warning if kindsToLoad is not a subset of the exported kinds.
|
||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
||||
logger.atWarning().log(
|
||||
"Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
||||
}
|
||||
// Load kinds from the backup, limited to those also in kindsToLoad (if it's present).
|
||||
ImmutableSet<String> exportedKindsToLoad =
|
||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
||||
String message = String.format("Datastore backup %s complete - ", backupName);
|
||||
if (exportedKindsToLoad.isEmpty()) {
|
||||
message += "no kinds to load into BigQuery.";
|
||||
} else {
|
||||
/** Enqueue a task for starting a backup load. */
|
||||
cloudTasksUtils.enqueue(
|
||||
UploadDatastoreBackupAction.QUEUE,
|
||||
cloudTasksUtils.createPostTask(
|
||||
UploadDatastoreBackupAction.PATH,
|
||||
Service.BACKEND.toString(),
|
||||
ImmutableMultimap.of(
|
||||
UploadDatastoreBackupAction.UPLOAD_BACKUP_ID_PARAM,
|
||||
backupId,
|
||||
UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM,
|
||||
backup.getExportFolderUrl(),
|
||||
UploadDatastoreBackupAction.UPLOAD_BACKUP_KINDS_PARAM,
|
||||
Joiner.on(',').join(exportedKindsToLoad))));
|
||||
message += "BigQuery load task enqueued.";
|
||||
}
|
||||
logger.atInfo().log(message);
|
||||
response.setPayload(message);
|
||||
}
|
||||
}
|
||||
@@ -15,49 +15,33 @@
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.base.Verify.verifyNotNull;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
import static google.registry.model.tld.Registries.getTldsOfType;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Suppliers;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.tld.Registry;
|
||||
import google.registry.model.tld.Registry.TldType;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.List;
|
||||
import java.util.function.Supplier;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* A mapreduce that exports the list of active domains on all real TLDs to Google Drive and GCS.
|
||||
* An action that exports the list of active domains on all real TLDs to Google Drive and GCS.
|
||||
*
|
||||
* <p>Each TLD's active domain names are exported as a newline-delimited flat text file with the
|
||||
* name TLD.txt into the domain-lists bucket. Note that this overwrites the files in place.
|
||||
@@ -70,11 +54,8 @@ import org.joda.time.DateTime;
|
||||
public class ExportDomainListsAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final int MAX_NUM_REDUCE_SHARDS = 100;
|
||||
public static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject DriveConnection driveConnection;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@@ -86,56 +67,44 @@ public class ExportDomainListsAction implements Runnable {
|
||||
public void run() {
|
||||
ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
|
||||
logger.atInfo().log("Exporting domain lists for TLDs %s.", realTlds);
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Export domain lists")
|
||||
.setModuleName("backend")
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(clock.nowUtc(), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsUtils),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
realTlds.forEach(
|
||||
tld -> {
|
||||
List<String> domains =
|
||||
tm().transact(
|
||||
() ->
|
||||
// Note that if we had "creationTime <= :now" in the condition (not
|
||||
// necessary as there is no pending creation, the order of deletionTime
|
||||
// and creationTime in the query would have been significant and it
|
||||
// should come after deletionTime. When Hibernate substitutes "now" it
|
||||
// will first validate that the **first** field that is to be compared
|
||||
// with it (deletionTime) is assignable from the substituted Java object
|
||||
// (click.nowUtc()). Since creationTime is a CreateAutoTimestamp, if it
|
||||
// comes first, we will need to substitute "now" with
|
||||
// CreateAutoTimestamp.create(clock.nowUtc()). This might look a bit
|
||||
// strange as the Java object type is clearly incompatible between the
|
||||
// two fields deletionTime (DateTime) and creationTime, yet they are
|
||||
// compared with the same "now". It is actually OK because in the end
|
||||
// Hibernate converts everything to SQL types (and Java field names to
|
||||
// SQL column names) to run the query. Both CreateAutoTimestamp and
|
||||
// DateTime are persisted as timestamp_z in SQL. It is only the
|
||||
// validation that compares the Java types, and only with the first
|
||||
// field that compares with the substituted value.
|
||||
jpaTm()
|
||||
.query(
|
||||
"SELECT fullyQualifiedDomainName FROM Domain "
|
||||
+ "WHERE tld = :tld "
|
||||
+ "AND deletionTime > :now "
|
||||
+ "ORDER by fullyQualifiedDomainName ASC",
|
||||
String.class)
|
||||
.setParameter("tld", tld)
|
||||
.setParameter("now", clock.nowUtc())
|
||||
.getResultList());
|
||||
String domainsList = Joiner.on("\n").join(domains);
|
||||
logger.atInfo().log(
|
||||
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsUtils);
|
||||
exportToDrive(tld, domainsList, driveConnection);
|
||||
});
|
||||
}
|
||||
realTlds.forEach(
|
||||
tld -> {
|
||||
List<String> domains =
|
||||
tm().transact(
|
||||
() ->
|
||||
// Note that if we had "creationTime <= :now" in the condition (not
|
||||
// necessary as there is no pending creation, the order of deletionTime
|
||||
// and creationTime in the query would have been significant and it
|
||||
// should come after deletionTime. When Hibernate substitutes "now" it
|
||||
// will first validate that the **first** field that is to be compared
|
||||
// with it (deletionTime) is assignable from the substituted Java object
|
||||
// (click.nowUtc()). Since creationTime is a CreateAutoTimestamp, if it
|
||||
// comes first, we will need to substitute "now" with
|
||||
// CreateAutoTimestamp.create(clock.nowUtc()). This might look a bit
|
||||
// strange as the Java object type is clearly incompatible between the
|
||||
// two fields deletionTime (DateTime) and creationTime, yet they are
|
||||
// compared with the same "now". It is actually OK because in the end
|
||||
// Hibernate converts everything to SQL types (and Java field names to
|
||||
// SQL column names) to run the query. Both CreateAutoTimestamp and
|
||||
// DateTime are persisted as timestamp_z in SQL. It is only the
|
||||
// validation that compares the Java types, and only with the first
|
||||
// field that compares with the substituted value.
|
||||
jpaTm()
|
||||
.query(
|
||||
"SELECT fullyQualifiedDomainName FROM Domain "
|
||||
+ "WHERE tld = :tld "
|
||||
+ "AND deletionTime > :now "
|
||||
+ "ORDER by fullyQualifiedDomainName ASC",
|
||||
String.class)
|
||||
.setParameter("tld", tld)
|
||||
.setParameter("now", clock.nowUtc())
|
||||
.getResultList());
|
||||
String domainsList = Joiner.on("\n").join(domains);
|
||||
logger.atInfo().log(
|
||||
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsUtils);
|
||||
exportToDrive(tld, domainsList, driveConnection);
|
||||
});
|
||||
}
|
||||
|
||||
protected static boolean exportToDrive(
|
||||
@@ -178,80 +147,4 @@ public class ExportDomainListsAction implements Runnable {
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static class ExportDomainListsMapper extends Mapper<DomainBase, String, String> {
|
||||
|
||||
private static final long serialVersionUID = -7312206212434039854L;
|
||||
|
||||
private final DateTime exportTime;
|
||||
private final ImmutableSet<String> realTlds;
|
||||
|
||||
ExportDomainListsMapper(DateTime exportTime, ImmutableSet<String> realTlds) {
|
||||
this.exportTime = exportTime;
|
||||
this.realTlds = realTlds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void map(DomainBase domain) {
|
||||
if (realTlds.contains(domain.getTld()) && isActive(domain, exportTime)) {
|
||||
emit(domain.getTld(), domain.getDomainName());
|
||||
getContext().incrementCounter(String.format("domains in tld %s", domain.getTld()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static class ExportDomainListsReducer extends Reducer<String, String, Void> {
|
||||
|
||||
private static final long serialVersionUID = 7035260977259119087L;
|
||||
|
||||
/** Allows overriding the default {@link DriveConnection} in tests. */
|
||||
@NonFinalForTesting
|
||||
private static Supplier<DriveConnection> driveConnectionSupplier =
|
||||
Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection());
|
||||
|
||||
private final String gcsBucket;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
/**
|
||||
* Non-serializable {@link DriveConnection} that will be created when an instance of {@link
|
||||
* ExportDomainListsReducer} is deserialized in a MR pipeline worker.
|
||||
*
|
||||
* <p>See {@link #readObject(ObjectInputStream)}.
|
||||
*/
|
||||
private transient DriveConnection driveConnection;
|
||||
|
||||
public ExportDomainListsReducer(String gcsBucket, GcsUtils gcsUtils) {
|
||||
this.gcsBucket = gcsBucket;
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException {
|
||||
is.defaultReadObject();
|
||||
driveConnection = driveConnectionSupplier.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reduce(String tld, ReducerInput<String> fqdns) {
|
||||
ImmutableList<String> domains = ImmutableList.sortedCopyOf(() -> fqdns);
|
||||
String domainsList = Joiner.on('\n').join(domains);
|
||||
logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsUtils)) {
|
||||
getContext().incrementCounter("domain lists successful written out to GCS");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to GCS");
|
||||
}
|
||||
if (exportToDrive(tld, domainsList, driveConnection)) {
|
||||
getContext().incrementCounter("domain lists successfully written out to Drive");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to Drive");
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static void setDriveConnectionForTesting(
|
||||
Supplier<DriveConnection> testDriveConnectionSupplier) {
|
||||
driveConnectionSupplier = testDriveConnectionSupplier;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
||||
import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
||||
import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
||||
import static google.registry.export.CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM;
|
||||
import static google.registry.export.CheckBackupAction.CHECK_BACKUP_NAME_PARAM;
|
||||
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
|
||||
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
|
||||
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
|
||||
import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM;
|
||||
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM;
|
||||
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_ID_PARAM;
|
||||
import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_KINDS_PARAM;
|
||||
import static google.registry.request.RequestParameters.extractRequiredHeader;
|
||||
import static google.registry.request.RequestParameters.extractRequiredParameter;
|
||||
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.request.Header;
|
||||
import google.registry.request.Parameter;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
/** Dagger module for data export tasks. */
|
||||
@Module
|
||||
public final class ExportRequestModule {
|
||||
|
||||
@Provides
|
||||
@Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM)
|
||||
static String provideUpdateSnapshotDatasetId(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, UPDATE_SNAPSHOT_DATASET_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM)
|
||||
static String provideUpdateSnapshotTableId(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, UPDATE_SNAPSHOT_TABLE_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(UPDATE_SNAPSHOT_KIND_PARAM)
|
||||
static String provideUpdateSnapshotKind(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, UPDATE_SNAPSHOT_KIND_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(UPDATE_SNAPSHOT_VIEWNAME_PARAM)
|
||||
static String provideUpdateSnapshotViewName(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, UPDATE_SNAPSHOT_VIEWNAME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(UPLOAD_BACKUP_FOLDER_PARAM)
|
||||
static String provideSnapshotUrlPrefix(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, UPLOAD_BACKUP_FOLDER_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(UPLOAD_BACKUP_ID_PARAM)
|
||||
static String provideLoadSnapshotId(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, UPLOAD_BACKUP_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(UPLOAD_BACKUP_KINDS_PARAM)
|
||||
static String provideLoadSnapshotKinds(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, UPLOAD_BACKUP_KINDS_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(CHECK_BACKUP_NAME_PARAM)
|
||||
static String provideCheckSnapshotName(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, CHECK_BACKUP_NAME_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(CHECK_BACKUP_KINDS_TO_LOAD_PARAM)
|
||||
static String provideCheckSnapshotKindsToLoad(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, CHECK_BACKUP_KINDS_TO_LOAD_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Header(CHAINED_TASK_QUEUE_HEADER)
|
||||
static String provideChainedTaskQueue(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, CHAINED_TASK_QUEUE_HEADER);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Header(JOB_ID_HEADER)
|
||||
static String provideJobId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, JOB_ID_HEADER);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Header(PROJECT_ID_HEADER)
|
||||
static String provideProjectId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, PROJECT_ID_HEADER);
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,7 @@ import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.groups.GroupsConnection;
|
||||
import google.registry.groups.GroupsConnection.Role;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarContact;
|
||||
import google.registry.model.registrar.RegistrarPoc;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
@@ -47,7 +47,7 @@ import javax.annotation.Nullable;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* Action that syncs changes to {@link RegistrarContact} entities with Google Groups.
|
||||
* Action that syncs changes to {@link RegistrarPoc} entities with Google Groups.
|
||||
*
|
||||
* <p>This uses the <a href="https://developers.google.com/admin-sdk/directory/">Directory API</a>.
|
||||
*/
|
||||
@@ -99,7 +99,7 @@ public final class SyncGroupMembersAction implements Runnable {
|
||||
* Returns the Google Groups email address for the given registrar ID and RegistrarContact.Type.
|
||||
*/
|
||||
public static String getGroupEmailAddressForContactType(
|
||||
String registrarId, RegistrarContact.Type type, String gSuiteDomainName) {
|
||||
String registrarId, RegistrarPoc.Type type, String gSuiteDomainName) {
|
||||
// Take the registrar's ID, make it lowercase, and remove all characters that aren't
|
||||
// alphanumeric, hyphens, or underscores.
|
||||
return String.format(
|
||||
@@ -171,18 +171,17 @@ public final class SyncGroupMembersAction implements Runnable {
|
||||
private void syncRegistrarContacts(Registrar registrar) {
|
||||
String groupKey = "";
|
||||
try {
|
||||
Set<RegistrarContact> registrarContacts = registrar.getContacts();
|
||||
Set<RegistrarPoc> registrarPocs = registrar.getContacts();
|
||||
long totalAdded = 0;
|
||||
long totalRemoved = 0;
|
||||
for (final RegistrarContact.Type type : RegistrarContact.Type.values()) {
|
||||
for (final RegistrarPoc.Type type : RegistrarPoc.Type.values()) {
|
||||
groupKey =
|
||||
getGroupEmailAddressForContactType(registrar.getRegistrarId(), type, gSuiteDomainName);
|
||||
Set<String> currentMembers = groupsConnection.getMembersOfGroup(groupKey);
|
||||
Set<String> desiredMembers =
|
||||
registrarContacts
|
||||
.stream()
|
||||
registrarPocs.stream()
|
||||
.filter(contact -> contact.getTypes().contains(type))
|
||||
.map(RegistrarContact::getEmailAddress)
|
||||
.map(RegistrarPoc::getEmailAddress)
|
||||
.collect(toImmutableSet());
|
||||
for (String email : Sets.difference(desiredMembers, currentMembers)) {
|
||||
groupsConnection.addMemberToGroup(groupKey, email, Role.MEMBER);
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.model.Table;
|
||||
import com.google.api.services.bigquery.model.TableReference;
|
||||
import com.google.api.services.bigquery.model.ViewDefinition;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.bigquery.CheckedBigquery;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.SqlTemplate;
|
||||
import java.io.IOException;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Update a well-known view to point at a certain Datastore snapshot table in BigQuery. */
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = UpdateSnapshotViewAction.PATH,
|
||||
method = POST,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class UpdateSnapshotViewAction implements Runnable {
|
||||
|
||||
/** Headers for passing parameters into the servlet. */
|
||||
static final String UPDATE_SNAPSHOT_DATASET_ID_PARAM = "dataset";
|
||||
|
||||
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
|
||||
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
|
||||
static final String UPDATE_SNAPSHOT_VIEWNAME_PARAM = "viewname";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
// For now this queue is shared by the backup workflows started by BackupDatastoreAction.
|
||||
// TODO(weiminyu): update queue name (snapshot->backup) after ExportSnapshot flow is removed.
|
||||
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
|
||||
|
||||
static final String PATH = "/_dr/task/updateSnapshotView"; // See web.xml.
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject
|
||||
@Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM)
|
||||
String datasetId;
|
||||
|
||||
@Inject
|
||||
@Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM)
|
||||
String tableId;
|
||||
|
||||
@Inject
|
||||
@Parameter(UPDATE_SNAPSHOT_KIND_PARAM)
|
||||
String kindName;
|
||||
|
||||
@Inject
|
||||
@Parameter(UPDATE_SNAPSHOT_VIEWNAME_PARAM)
|
||||
String viewName;
|
||||
|
||||
@Inject
|
||||
@Config("projectId")
|
||||
String projectId;
|
||||
|
||||
@Inject CheckedBigquery checkedBigquery;
|
||||
|
||||
@Inject
|
||||
UpdateSnapshotViewAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
SqlTemplate sqlTemplate =
|
||||
SqlTemplate.create(
|
||||
"#standardSQL\nSELECT * FROM `%PROJECT%.%SOURCE_DATASET%.%SOURCE_TABLE%`");
|
||||
updateSnapshotView(datasetId, tableId, kindName, viewName, sqlTemplate);
|
||||
} catch (Throwable e) {
|
||||
throw new InternalServerErrorException(
|
||||
String.format("Could not update snapshot view %s for table %s", viewName, tableId), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateSnapshotView(
|
||||
String sourceDatasetId,
|
||||
String sourceTableId,
|
||||
String kindName,
|
||||
String viewDataset,
|
||||
SqlTemplate viewQueryTemplate)
|
||||
throws IOException {
|
||||
|
||||
Bigquery bigquery = checkedBigquery.ensureDataSetExists(projectId, viewDataset);
|
||||
updateTable(
|
||||
bigquery,
|
||||
new Table()
|
||||
.setTableReference(
|
||||
new TableReference()
|
||||
.setProjectId(projectId)
|
||||
.setDatasetId(viewDataset)
|
||||
.setTableId(kindName))
|
||||
.setView(
|
||||
new ViewDefinition()
|
||||
.setUseLegacySql(false)
|
||||
.setQuery(
|
||||
viewQueryTemplate
|
||||
.put("PROJECT", projectId)
|
||||
.put("SOURCE_DATASET", sourceDatasetId)
|
||||
.put("SOURCE_TABLE", sourceTableId)
|
||||
.build())));
|
||||
|
||||
logger.atInfo().log(
|
||||
"Updated view [%s:%s.%s] to point at snapshot table [%s:%s.%s].",
|
||||
projectId, viewDataset, kindName, projectId, sourceDatasetId, sourceTableId);
|
||||
}
|
||||
|
||||
private static void updateTable(Bigquery bigquery, Table table) throws IOException {
|
||||
TableReference ref = table.getTableReference();
|
||||
try {
|
||||
bigquery
|
||||
.tables()
|
||||
.update(ref.getProjectId(), ref.getDatasetId(), ref.getTableId(), table)
|
||||
.execute();
|
||||
} catch (GoogleJsonResponseException e) {
|
||||
if (e.getDetails() != null && e.getDetails().getCode() == 404) {
|
||||
bigquery.tables().insert(ref.getProjectId(), ref.getDatasetId(), table).execute();
|
||||
} else {
|
||||
logger.atWarning().withCause(e).log("UpdateSnapshotViewAction errored out.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.model.Job;
|
||||
import com.google.api.services.bigquery.model.JobConfiguration;
|
||||
import com.google.api.services.bigquery.model.JobConfigurationLoad;
|
||||
import com.google.api.services.bigquery.model.JobReference;
|
||||
import com.google.api.services.bigquery.model.TableReference;
|
||||
import com.google.cloud.tasks.v2.Task;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.HttpHeaders;
|
||||
import com.google.common.net.MediaType;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.util.Timestamps;
|
||||
import google.registry.bigquery.BigqueryUtils.SourceFormat;
|
||||
import google.registry.bigquery.BigqueryUtils.WriteDisposition;
|
||||
import google.registry.bigquery.CheckedBigquery;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Action.Service;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.CloudTasksUtils;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectOutputStream;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Action to load a Datastore backup from Google Cloud Storage into BigQuery. */
|
||||
@Action(
|
||||
service = Action.Service.BACKEND,
|
||||
path = UploadDatastoreBackupAction.PATH,
|
||||
method = POST,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class UploadDatastoreBackupAction implements Runnable {
|
||||
|
||||
/** Parameter names for passing parameters into the servlet. */
|
||||
static final String UPLOAD_BACKUP_ID_PARAM = "id";
|
||||
|
||||
static final String UPLOAD_BACKUP_FOLDER_PARAM = "folder";
|
||||
static final String UPLOAD_BACKUP_KINDS_PARAM = "kinds";
|
||||
|
||||
static final String BACKUP_DATASET = "datastore_backups";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||
|
||||
static final String LATEST_BACKUP_VIEW_NAME = "latest_datastore_export";
|
||||
|
||||
static final String PATH = "/_dr/task/uploadDatastoreBackup"; // See web.xml.
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject CheckedBigquery checkedBigquery;
|
||||
@Inject CloudTasksUtils cloudTasksUtils;
|
||||
@Inject Clock clock;
|
||||
|
||||
@Inject @Config("projectId") String projectId;
|
||||
|
||||
@Inject
|
||||
@Parameter(UPLOAD_BACKUP_FOLDER_PARAM)
|
||||
String backupFolderUrl;
|
||||
|
||||
@Inject
|
||||
@Parameter(UPLOAD_BACKUP_ID_PARAM)
|
||||
String backupId;
|
||||
|
||||
@Inject
|
||||
@Parameter(UPLOAD_BACKUP_KINDS_PARAM)
|
||||
String backupKinds;
|
||||
|
||||
@Inject
|
||||
UploadDatastoreBackupAction() {}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
String message = uploadBackup(backupId, backupFolderUrl, Splitter.on(',').split(backupKinds));
|
||||
logger.atInfo().log("Loaded backup successfully: %s", message);
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log("Error loading backup.");
|
||||
if (e instanceof IllegalArgumentException) {
|
||||
throw new BadRequestException("Error calling load backup: " + e.getMessage(), e);
|
||||
} else {
|
||||
throw new InternalServerErrorException(
|
||||
"Error loading backup: " + firstNonNull(e.getMessage(), e.toString()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String uploadBackup(String backupId, String backupFolderUrl, Iterable<String> kinds)
|
||||
throws IOException {
|
||||
Bigquery bigquery = checkedBigquery.ensureDataSetExists(projectId, BACKUP_DATASET);
|
||||
String loadMessage =
|
||||
String.format("Loading Datastore backup %s from %s...", backupId, backupFolderUrl);
|
||||
logger.atInfo().log(loadMessage);
|
||||
|
||||
String sanitizedBackupId = sanitizeForBigquery(backupId);
|
||||
StringBuilder builder = new StringBuilder(loadMessage + "\n");
|
||||
builder.append("Load jobs:\n");
|
||||
|
||||
for (String kindName : kinds) {
|
||||
String jobId = String.format("load-backup-%s-%s", sanitizedBackupId, kindName);
|
||||
JobReference jobRef = new JobReference().setProjectId(projectId).setJobId(jobId);
|
||||
String sourceUri = getBackupInfoFileForKind(backupFolderUrl, kindName);
|
||||
String tableId = String.format("%s_%s", sanitizedBackupId, kindName);
|
||||
|
||||
// Launch the load job.
|
||||
Job job = makeLoadJob(jobRef, sourceUri, tableId);
|
||||
bigquery.jobs().insert(projectId, job).execute();
|
||||
|
||||
// Serialize the chainedTask into a byte array to put in the task payload.
|
||||
ByteArrayOutputStream taskBytes = new ByteArrayOutputStream();
|
||||
new ObjectOutputStream(taskBytes)
|
||||
.writeObject(
|
||||
cloudTasksUtils.createPostTask(
|
||||
UpdateSnapshotViewAction.PATH,
|
||||
Service.BACKEND.toString(),
|
||||
ImmutableMultimap.of(
|
||||
UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM,
|
||||
BACKUP_DATASET,
|
||||
UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM,
|
||||
tableId,
|
||||
UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM,
|
||||
kindName,
|
||||
UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM,
|
||||
LATEST_BACKUP_VIEW_NAME)));
|
||||
|
||||
// Enqueues a task to poll for the success or failure of the referenced BigQuery job and to
|
||||
// launch the provided task in the specified queue if the job succeeds.
|
||||
cloudTasksUtils.enqueue(
|
||||
BigqueryPollJobAction.QUEUE,
|
||||
Task.newBuilder()
|
||||
.setAppEngineHttpRequest(
|
||||
cloudTasksUtils
|
||||
.createPostTask(BigqueryPollJobAction.PATH, Service.BACKEND.toString(), null)
|
||||
.getAppEngineHttpRequest()
|
||||
.toBuilder()
|
||||
.putHeaders(BigqueryPollJobAction.PROJECT_ID_HEADER, jobRef.getProjectId())
|
||||
.putHeaders(BigqueryPollJobAction.JOB_ID_HEADER, jobRef.getJobId())
|
||||
.putHeaders(
|
||||
BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER,
|
||||
UpdateSnapshotViewAction.QUEUE)
|
||||
// need to include CONTENT_TYPE in header when body is not empty
|
||||
.putHeaders(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
|
||||
.setBody(ByteString.copyFrom(taskBytes.toByteArray()))
|
||||
.build())
|
||||
.setScheduleTime(
|
||||
Timestamps.fromMillis(
|
||||
clock.nowUtc().plus(BigqueryPollJobAction.POLL_COUNTDOWN).getMillis()))
|
||||
.build());
|
||||
|
||||
builder.append(String.format(" - %s:%s\n", projectId, jobId));
|
||||
logger.atInfo().log("Submitted load job %s:%s.", projectId, jobId);
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private static String sanitizeForBigquery(String backupId) {
|
||||
return backupId.replaceAll("[^a-zA-Z0-9_]", "_");
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static String getBackupInfoFileForKind(String backupFolderUrl, String kindName) {
|
||||
return Joiner.on('/')
|
||||
.join(
|
||||
backupFolderUrl,
|
||||
"all_namespaces",
|
||||
String.format("kind_%s", kindName),
|
||||
String.format("all_namespaces_kind_%s.%s", kindName, "export_metadata"));
|
||||
}
|
||||
|
||||
private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
|
||||
TableReference tableReference =
|
||||
new TableReference()
|
||||
.setProjectId(jobRef.getProjectId())
|
||||
.setDatasetId(BACKUP_DATASET)
|
||||
.setTableId(tableId);
|
||||
return new Job()
|
||||
.setJobReference(jobRef)
|
||||
.setConfiguration(new JobConfiguration()
|
||||
.setLoad(new JobConfigurationLoad()
|
||||
.setWriteDisposition(WriteDisposition.WRITE_EMPTY.toString())
|
||||
.setSourceFormat(SourceFormat.DATASTORE_BACKUP.toString())
|
||||
.setSourceUris(ImmutableList.of(sourceUri))
|
||||
.setDestinationTable(tableReference)));
|
||||
}
|
||||
}
|
||||
@@ -1,279 +0,0 @@
|
||||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export.datastore;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
import com.google.api.client.googleapis.services.json.AbstractGoogleJsonClient;
|
||||
import com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest;
|
||||
import com.google.api.client.http.HttpRequestInitializer;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.json.GenericJson;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.client.util.Key;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Java client to <a href="https://cloud.google.com/datastore/docs/reference/admin/rest/">Cloud
|
||||
* Datastore Admin REST API</a>.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class DatastoreAdmin extends AbstractGoogleJsonClient {
|
||||
|
||||
private static final String ROOT_URL = "https://datastore.googleapis.com/v1/";
|
||||
private static final String SERVICE_PATH = "";
|
||||
|
||||
// GCP project that this instance is associated with.
|
||||
private final String projectId;
|
||||
|
||||
protected DatastoreAdmin(Builder builder) {
|
||||
super(builder);
|
||||
this.projectId = checkNotNull(builder.projectId, "GCP projectId missing.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an {@link Export} request that starts exporting all Cloud Datastore databases owned by
|
||||
* the GCP project identified by {@link #projectId}.
|
||||
*
|
||||
* <p>Typical usage is:
|
||||
*
|
||||
* <pre>
|
||||
* {@code Export export = datastoreAdmin.export(parameters ...);}
|
||||
* {@code Operation operation = export.execute();}
|
||||
* {@code while (!operation.isSuccessful()) { ...}}
|
||||
* </pre>
|
||||
*
|
||||
* <p>Please see the <a
|
||||
* href="https://cloud.google.com/datastore/docs/reference/admin/rest/v1/projects/export">API
|
||||
* specification of the export method for details</a>.
|
||||
*
|
||||
* <p>The following undocumented behaviors with regard to {@code outputUrlPrefix} have been
|
||||
* observed:
|
||||
*
|
||||
* <ul>
|
||||
* <li>If outputUrlPrefix refers to a GCS bucket, exported data will be nested deeper in the
|
||||
* bucket with a timestamped path. This is useful when periodical backups are desired
|
||||
* <li>If outputUrlPrefix is a already a nested path in a GCS bucket, exported data will be put
|
||||
* under this path. This means that a nested path is not reusable, since the export process
|
||||
* by default would not overwrite existing files.
|
||||
* </ul>
|
||||
*
|
||||
* @param outputUrlPrefix the full resource URL of the external storage location
|
||||
* @param kinds the datastore 'kinds' to be exported. If empty, all kinds will be exported
|
||||
*/
|
||||
public Export export(String outputUrlPrefix, Collection<String> kinds) {
|
||||
return new Export(new ExportRequest(outputUrlPrefix, kinds));
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports the entire backup specified by {@code backupUrl} back to Cloud Datastore.
|
||||
*
|
||||
* <p>A successful backup restores deleted entities and reverts updates to existing entities since
|
||||
* the backup time. However, it does not affect newly added entities.
|
||||
*/
|
||||
public Import importBackup(String backupUrl) {
|
||||
return new Import(new ImportRequest(backupUrl, ImmutableList.of()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports the backup specified by {@code backupUrl} back to Cloud Datastore. Only entities whose
|
||||
* types are included in {@code kinds} are imported.
|
||||
*
|
||||
* @see #importBackup(String)
|
||||
*/
|
||||
public Import importBackup(String backupUrl, Collection<String> kinds) {
|
||||
return new Import(new ImportRequest(backupUrl, kinds));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link Get} request that retrieves the details of an export or import {@link
|
||||
* Operation}.
|
||||
*
|
||||
* @param operationName name of the {@code Operation} as returned by an export or import request
|
||||
*/
|
||||
public Get get(String operationName) {
|
||||
return new Get(operationName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link ListOperations} request that retrieves all export or import {@link Operation
|
||||
* operations} matching {@code filter}.
|
||||
*
|
||||
* <p>Sample usage: find all operations started after 2018-10-31 00:00:00 UTC and has stopped:
|
||||
*
|
||||
* <pre>
|
||||
* {@code String filter = "metadata.common.startTime>\"2018-10-31T0:0:0Z\" AND done=true";}
|
||||
* {@code List<Operation> operations = datastoreAdmin.list(filter);}
|
||||
* </pre>
|
||||
*
|
||||
* <p>Please refer to {@link Operation} for how to reference operation properties.
|
||||
*/
|
||||
public ListOperations list(String filter) {
|
||||
checkArgument(!Strings.isNullOrEmpty(filter), "Filter must not be null or empty.");
|
||||
return new ListOperations(Optional.of(filter));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link ListOperations} request that retrieves all export or import {@link Operation *
|
||||
* operations}.
|
||||
*/
|
||||
public ListOperations listAll() {
|
||||
return new ListOperations(Optional.empty());
|
||||
}
|
||||
|
||||
/** Builder for {@link DatastoreAdmin}. */
|
||||
public static class Builder extends AbstractGoogleJsonClient.Builder {
|
||||
|
||||
private String projectId;
|
||||
|
||||
public Builder(
|
||||
HttpTransport httpTransport,
|
||||
JsonFactory jsonFactory,
|
||||
HttpRequestInitializer httpRequestInitializer) {
|
||||
super(httpTransport, jsonFactory, ROOT_URL, SERVICE_PATH, httpRequestInitializer, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Builder setApplicationName(String applicationName) {
|
||||
return (Builder) super.setApplicationName(applicationName);
|
||||
}
|
||||
|
||||
/** Sets the GCP project ID of the Cloud Datastore databases being managed. */
|
||||
public Builder setProjectId(String projectId) {
|
||||
this.projectId = projectId;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatastoreAdmin build() {
|
||||
return new DatastoreAdmin(this);
|
||||
}
|
||||
}
|
||||
|
||||
/** A request to export Cloud Datastore databases. */
|
||||
public class Export extends DatastoreAdminRequest<Operation> {
|
||||
|
||||
Export(ExportRequest exportRequest) {
|
||||
super(
|
||||
DatastoreAdmin.this,
|
||||
"POST",
|
||||
"projects/{projectId}:export",
|
||||
exportRequest,
|
||||
Operation.class);
|
||||
set("projectId", projectId);
|
||||
}
|
||||
}
|
||||
|
||||
/** A request to restore an backup to a Cloud Datastore database. */
|
||||
public class Import extends DatastoreAdminRequest<Operation> {
|
||||
|
||||
Import(ImportRequest importRequest) {
|
||||
super(
|
||||
DatastoreAdmin.this,
|
||||
"POST",
|
||||
"projects/{projectId}:import",
|
||||
importRequest,
|
||||
Operation.class);
|
||||
set("projectId", projectId);
|
||||
}
|
||||
}
|
||||
|
||||
/** A request to retrieve details of an export or import operation. */
|
||||
public class Get extends DatastoreAdminRequest<Operation> {
|
||||
|
||||
Get(String operationName) {
|
||||
super(DatastoreAdmin.this, "GET", operationName, null, Operation.class);
|
||||
}
|
||||
}
|
||||
|
||||
/** A request to retrieve all export or import operations matching a given filter. */
|
||||
public class ListOperations extends DatastoreAdminRequest<Operation.OperationList> {
|
||||
|
||||
ListOperations(Optional<String> filter) {
|
||||
super(
|
||||
DatastoreAdmin.this,
|
||||
"GET",
|
||||
"projects/{projectId}/operations",
|
||||
null,
|
||||
Operation.OperationList.class);
|
||||
set("projectId", projectId);
|
||||
filter.ifPresent(f -> set("filter", f));
|
||||
}
|
||||
}
|
||||
|
||||
/** Base class of all DatastoreAdmin requests. */
|
||||
abstract static class DatastoreAdminRequest<T> extends AbstractGoogleJsonClientRequest<T> {
|
||||
/**
|
||||
* @param client Google JSON client
|
||||
* @param requestMethod HTTP Method
|
||||
* @param uriTemplate URI template for the path relative to the base URL. If it starts with a
|
||||
* "/" the base path from the base URL will be stripped out. The URI template can also be a
|
||||
* full URL. URI template expansion is done using {@link
|
||||
* com.google.api.client.http.UriTemplate#expand(String, String, Object, boolean)}
|
||||
* @param jsonContent POJO that can be serialized into JSON content or {@code null} for none
|
||||
* @param responseClass response class to parse into
|
||||
*/
|
||||
protected DatastoreAdminRequest(
|
||||
DatastoreAdmin client,
|
||||
String requestMethod,
|
||||
String uriTemplate,
|
||||
Object jsonContent,
|
||||
Class<T> responseClass) {
|
||||
super(client, requestMethod, uriTemplate, jsonContent, responseClass);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Model object that describes the JSON content in an export request.
|
||||
*
|
||||
* <p>Please note that some properties defined in the API are excluded, e.g., {@code databaseId}
|
||||
* (not supported by Cloud Datastore) and labels (not used by Domain Registry).
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
static class ExportRequest extends GenericJson {
|
||||
@Key private final String outputUrlPrefix;
|
||||
@Key private final EntityFilter entityFilter;
|
||||
|
||||
ExportRequest(String outputUrlPrefix, Collection<String> kinds) {
|
||||
checkNotNull(outputUrlPrefix, "outputUrlPrefix");
|
||||
checkArgument(!kinds.isEmpty(), "kinds must not be empty");
|
||||
this.outputUrlPrefix = outputUrlPrefix;
|
||||
this.entityFilter = new EntityFilter(kinds);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Model object that describes the JSON content in an export request.
|
||||
*
|
||||
* <p>Please note that some properties defined in the API are excluded, e.g., {@code databaseId}
|
||||
* (not supported by Cloud Datastore) and labels (not used by Domain Registry).
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
static class ImportRequest extends GenericJson {
|
||||
@Key private final String inputUrl;
|
||||
@Key private final EntityFilter entityFilter;
|
||||
|
||||
ImportRequest(String inputUrl, Collection<String> kinds) {
|
||||
checkNotNull(inputUrl, "outputUrlPrefix");
|
||||
this.inputUrl = inputUrl;
|
||||
this.entityFilter = new EntityFilter(kinds);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export.datastore;
|
||||
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import google.registry.config.CredentialModule;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.util.GoogleCredentialsBundle;
|
||||
import javax.inject.Singleton;
|
||||
|
||||
/** Dagger module that configures provision of {@link DatastoreAdmin}. */
|
||||
@Module
|
||||
@DeleteAfterMigration
|
||||
public abstract class DatastoreAdminModule {
|
||||
|
||||
@Singleton
|
||||
@Provides
|
||||
static DatastoreAdmin provideDatastoreAdmin(
|
||||
@CredentialModule.DefaultCredential GoogleCredentialsBundle credentialsBundle,
|
||||
@RegistryConfig.Config("projectId") String projectId) {
|
||||
return new DatastoreAdmin.Builder(
|
||||
credentialsBundle.getHttpTransport(),
|
||||
credentialsBundle.getJsonFactory(),
|
||||
credentialsBundle.getHttpRequestInitializer())
|
||||
.setApplicationName(projectId)
|
||||
.setProjectId(projectId)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export.datastore;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
import com.google.api.client.json.GenericJson;
|
||||
import com.google.api.client.util.Key;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Model object that describes the Cloud Datastore 'kinds' to be exported or imported. The JSON form
|
||||
* of this type is found in export/import requests and responses.
|
||||
*
|
||||
* <p>Please note that properties not used by Domain Registry are not included, e.g., {@code
|
||||
* namespaceIds}.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class EntityFilter extends GenericJson {
|
||||
|
||||
@Key private List<String> kinds = ImmutableList.of();
|
||||
|
||||
/** For JSON deserialization. */
|
||||
public EntityFilter() {}
|
||||
|
||||
EntityFilter(Collection<String> kinds) {
|
||||
checkNotNull(kinds, "kinds");
|
||||
this.kinds = ImmutableList.copyOf(kinds);
|
||||
}
|
||||
|
||||
List<String> getKinds() {
|
||||
return ImmutableList.copyOf(kinds);
|
||||
}
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.export.datastore;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
|
||||
import com.google.api.client.json.GenericJson;
|
||||
import com.google.api.client.util.Key;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.export.datastore.DatastoreAdmin.Get;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import javax.annotation.Nullable;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.Duration;
|
||||
|
||||
/**
|
||||
* Model object that describes the details of an export or import operation in Cloud Datastore.
|
||||
*
|
||||
* <p>{@link Operation} instances are parsed from the JSON payload in Datastore response messages.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class Operation extends GenericJson {
|
||||
|
||||
private static final String STATE_SUCCESS = "SUCCESSFUL";
|
||||
private static final String STATE_PROCESSING = "PROCESSING";
|
||||
|
||||
@Key private String name;
|
||||
@Key private Metadata metadata;
|
||||
@Key private boolean done;
|
||||
|
||||
/** For JSON deserialization. */
|
||||
public Operation() {}
|
||||
|
||||
/** Returns the name of this operation, which may be used in a {@link Get} request. */
|
||||
public String getName() {
|
||||
checkState(name != null, "Name must not be null.");
|
||||
return name;
|
||||
}
|
||||
|
||||
public boolean isExport() {
|
||||
return !isNullOrEmpty(getExportFolderUrl());
|
||||
}
|
||||
|
||||
public boolean isImport() {
|
||||
return !isNullOrEmpty(getMetadata().getInputUrl());
|
||||
}
|
||||
|
||||
public boolean isDone() {
|
||||
return done;
|
||||
}
|
||||
|
||||
private String getState() {
|
||||
return getMetadata().getCommonMetadata().getState();
|
||||
}
|
||||
|
||||
public boolean isSuccessful() {
|
||||
return getState().equals(STATE_SUCCESS);
|
||||
}
|
||||
|
||||
public boolean isProcessing() {
|
||||
return getState().equals(STATE_PROCESSING);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the elapsed time since starting if this operation is still running, or the total
|
||||
* running time if this operation has completed.
|
||||
*/
|
||||
public Duration getRunningTime(Clock clock) {
|
||||
return new Duration(
|
||||
getStartTime(), getMetadata().getCommonMetadata().getEndTime().orElse(clock.nowUtc()));
|
||||
}
|
||||
|
||||
public DateTime getStartTime() {
|
||||
return getMetadata().getCommonMetadata().getStartTime();
|
||||
}
|
||||
|
||||
public ImmutableSet<String> getKinds() {
|
||||
return ImmutableSet.copyOf(getMetadata().getEntityFilter().getKinds());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the URL to the GCS folder that holds the exported data. This folder is created by
|
||||
* Datastore and is under the {@code outputUrlPrefix} set to {@linkplain
|
||||
* DatastoreAdmin#export(String, java.util.Collection) the export request}.
|
||||
*
|
||||
* @throws IllegalStateException if this is not an export operation
|
||||
*/
|
||||
public String getExportFolderUrl() {
|
||||
return getMetadata().getOutputUrlPrefix();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the last segment of the {@linkplain #getExportFolderUrl() export folder URL} which can
|
||||
* be used as unique identifier of this export operation. This is a better ID than the {@linkplain
|
||||
* #getName() operation name}, which is opaque.
|
||||
*
|
||||
* @throws IllegalStateException if this is not an export operation
|
||||
*/
|
||||
public String getExportId() {
|
||||
String exportFolderUrl = getExportFolderUrl();
|
||||
return exportFolderUrl.substring(exportFolderUrl.lastIndexOf('/') + 1);
|
||||
}
|
||||
|
||||
public String getProgress() {
|
||||
StringBuilder result = new StringBuilder();
|
||||
Progress progress = getMetadata().getProgressBytes();
|
||||
if (progress != null) {
|
||||
result.append(
|
||||
String.format(" [%s/%s bytes]", progress.workCompleted, progress.workEstimated));
|
||||
}
|
||||
progress = getMetadata().getProgressEntities();
|
||||
if (progress != null) {
|
||||
result.append(
|
||||
String.format(" [%s/%s entities]", progress.workCompleted, progress.workEstimated));
|
||||
}
|
||||
if (result.length() == 0) {
|
||||
return "Progress: N/A";
|
||||
}
|
||||
return "Progress:" + result;
|
||||
}
|
||||
|
||||
private Metadata getMetadata() {
|
||||
checkState(metadata != null, "Response metadata missing.");
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/** Models the common metadata properties of all operations. */
|
||||
public static class CommonMetadata extends GenericJson {
|
||||
|
||||
@Key private String startTime;
|
||||
@Key @Nullable private String endTime;
|
||||
@Key private String operationType;
|
||||
@Key private String state;
|
||||
|
||||
public CommonMetadata() {}
|
||||
|
||||
String getOperationType() {
|
||||
checkState(!isNullOrEmpty(operationType), "operationType may not be null or empty");
|
||||
return operationType;
|
||||
}
|
||||
|
||||
String getState() {
|
||||
checkState(!isNullOrEmpty(state), "state may not be null or empty");
|
||||
return state;
|
||||
}
|
||||
|
||||
DateTime getStartTime() {
|
||||
checkState(startTime != null, "StartTime missing.");
|
||||
return DateTime.parse(startTime);
|
||||
}
|
||||
|
||||
Optional<DateTime> getEndTime() {
|
||||
return Optional.ofNullable(endTime).map(DateTime::parse);
|
||||
}
|
||||
}
|
||||
|
||||
/** Models the metadata of a Cloud Datatore export or import operation. */
|
||||
public static class Metadata extends GenericJson {
|
||||
@Key("common")
|
||||
private CommonMetadata commonMetadata;
|
||||
|
||||
@Key private Progress progressEntities;
|
||||
@Key private Progress progressBytes;
|
||||
@Key private EntityFilter entityFilter;
|
||||
@Key private String inputUrl;
|
||||
@Key private String outputUrlPrefix;
|
||||
|
||||
public Metadata() {}
|
||||
|
||||
CommonMetadata getCommonMetadata() {
|
||||
checkState(commonMetadata != null, "CommonMetadata field is null.");
|
||||
return commonMetadata;
|
||||
}
|
||||
|
||||
public Progress getProgressEntities() {
|
||||
return progressEntities;
|
||||
}
|
||||
|
||||
public Progress getProgressBytes() {
|
||||
return progressBytes;
|
||||
}
|
||||
|
||||
public EntityFilter getEntityFilter() {
|
||||
return entityFilter;
|
||||
}
|
||||
|
||||
public String getInputUrl() {
|
||||
return checkUrls().inputUrl;
|
||||
}
|
||||
|
||||
public String getOutputUrlPrefix() {
|
||||
return checkUrls().outputUrlPrefix;
|
||||
}
|
||||
|
||||
Metadata checkUrls() {
|
||||
checkState(
|
||||
isNullOrEmpty(inputUrl) || isNullOrEmpty(outputUrlPrefix),
|
||||
"inputUrl and outputUrlPrefix must not be both present");
|
||||
checkState(
|
||||
!isNullOrEmpty(inputUrl) || !isNullOrEmpty(outputUrlPrefix),
|
||||
"inputUrl and outputUrlPrefix must not be both missing");
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
/** Progress of an export or import operation. */
|
||||
public static class Progress extends GenericJson {
|
||||
@Key private long workCompleted;
|
||||
@Key private long workEstimated;
|
||||
|
||||
public Progress() {}
|
||||
|
||||
long getWorkCompleted() {
|
||||
return workCompleted;
|
||||
}
|
||||
|
||||
public long getWorkEstimated() {
|
||||
return workEstimated;
|
||||
}
|
||||
}
|
||||
|
||||
/** List of {@link Operation Operations}. */
|
||||
public static class OperationList extends GenericJson {
|
||||
@Key private List<Operation> operations;
|
||||
|
||||
/** For JSON deserialization. */
|
||||
public OperationList() {}
|
||||
|
||||
ImmutableList<Operation> toList() {
|
||||
return ImmutableList.copyOf(operations);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,15 +17,14 @@ package google.registry.export.sheet;
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static google.registry.model.common.Cursor.CursorType.SYNC_REGISTRAR_SHEET;
|
||||
import static google.registry.model.registrar.RegistrarContact.Type.ABUSE;
|
||||
import static google.registry.model.registrar.RegistrarContact.Type.ADMIN;
|
||||
import static google.registry.model.registrar.RegistrarContact.Type.BILLING;
|
||||
import static google.registry.model.registrar.RegistrarContact.Type.LEGAL;
|
||||
import static google.registry.model.registrar.RegistrarContact.Type.MARKETING;
|
||||
import static google.registry.model.registrar.RegistrarContact.Type.TECH;
|
||||
import static google.registry.model.registrar.RegistrarContact.Type.WHOIS;
|
||||
import static google.registry.model.registrar.RegistrarPoc.Type.ABUSE;
|
||||
import static google.registry.model.registrar.RegistrarPoc.Type.ADMIN;
|
||||
import static google.registry.model.registrar.RegistrarPoc.Type.BILLING;
|
||||
import static google.registry.model.registrar.RegistrarPoc.Type.LEGAL;
|
||||
import static google.registry.model.registrar.RegistrarPoc.Type.MARKETING;
|
||||
import static google.registry.model.registrar.RegistrarPoc.Type.TECH;
|
||||
import static google.registry.model.registrar.RegistrarPoc.Type.WHOIS;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
@@ -36,7 +35,7 @@ import com.google.common.collect.Ordering;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarAddress;
|
||||
import google.registry.model.registrar.RegistrarContact;
|
||||
import google.registry.model.registrar.RegistrarPoc;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.DateTimeUtils;
|
||||
import java.io.IOException;
|
||||
@@ -63,8 +62,7 @@ class SyncRegistrarsSheet {
|
||||
*/
|
||||
boolean wereRegistrarsModified() {
|
||||
Optional<Cursor> cursor =
|
||||
transactIfJpaTm(
|
||||
() -> tm().loadByKeyIfPresent(Cursor.createGlobalVKey(SYNC_REGISTRAR_SHEET)));
|
||||
tm().transact(() -> tm().loadByKeyIfPresent(Cursor.createGlobalVKey(SYNC_REGISTRAR_SHEET)));
|
||||
DateTime lastUpdateTime = !cursor.isPresent() ? START_OF_TIME : cursor.get().getCursorTime();
|
||||
for (Registrar registrar : Registrar.loadAllCached()) {
|
||||
if (DateTimeUtils.isAtOrAfter(registrar.getLastUpdateTime(), lastUpdateTime)) {
|
||||
@@ -92,7 +90,7 @@ class SyncRegistrarsSheet {
|
||||
.map(
|
||||
registrar -> {
|
||||
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
|
||||
ImmutableSortedSet<RegistrarContact> contacts = registrar.getContacts();
|
||||
ImmutableSortedSet<RegistrarPoc> contacts = registrar.getContacts();
|
||||
RegistrarAddress address =
|
||||
firstNonNull(
|
||||
registrar.getLocalizedAddress(),
|
||||
@@ -130,10 +128,10 @@ class SyncRegistrarsSheet {
|
||||
builder.put("billingContacts", convertContacts(contacts, byType(BILLING)));
|
||||
builder.put(
|
||||
"contactsMarkedAsWhoisAdmin",
|
||||
convertContacts(contacts, RegistrarContact::getVisibleInWhoisAsAdmin));
|
||||
convertContacts(contacts, RegistrarPoc::getVisibleInWhoisAsAdmin));
|
||||
builder.put(
|
||||
"contactsMarkedAsWhoisTech",
|
||||
convertContacts(contacts, RegistrarContact::getVisibleInWhoisAsTech));
|
||||
convertContacts(contacts, RegistrarPoc::getVisibleInWhoisAsTech));
|
||||
builder.put("emailAddress", convert(registrar.getEmailAddress()));
|
||||
builder.put("address.street", convert(address.getStreet()));
|
||||
builder.put("address.city", convert(address.getCity()));
|
||||
@@ -158,10 +156,10 @@ class SyncRegistrarsSheet {
|
||||
}
|
||||
|
||||
private static String convertContacts(
|
||||
Iterable<RegistrarContact> contacts, Predicate<RegistrarContact> filter) {
|
||||
Iterable<RegistrarPoc> contacts, Predicate<RegistrarPoc> filter) {
|
||||
StringBuilder result = new StringBuilder();
|
||||
boolean first = true;
|
||||
for (RegistrarContact contact : contacts) {
|
||||
for (RegistrarPoc contact : contacts) {
|
||||
if (!filter.test(contact)) {
|
||||
continue;
|
||||
}
|
||||
@@ -175,7 +173,7 @@ class SyncRegistrarsSheet {
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
private static Predicate<RegistrarContact> byType(final RegistrarContact.Type type) {
|
||||
private static Predicate<RegistrarPoc> byType(final RegistrarPoc.Type type) {
|
||||
return contact -> contact.getTypes().contains(type);
|
||||
}
|
||||
|
||||
|
||||
@@ -84,5 +84,11 @@ public class EppToolAction implements Runnable {
|
||||
static String provideClientId(HttpServletRequest req) {
|
||||
return extractRequiredParameter(req, "clientId");
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter("dryRun")
|
||||
static boolean provideIsDryRun(HttpServletRequest req) {
|
||||
return extractBooleanParameter(req, "dryRun");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ public final class FlowUtils {
|
||||
}
|
||||
}
|
||||
|
||||
/** Persists the saves and deletes in an {@link EntityChanges} to Datastore. */
|
||||
/** Persists the saves and deletes in an {@link EntityChanges} to the DB. */
|
||||
public static void persistEntityChanges(EntityChanges entityChanges) {
|
||||
tm().putAll(entityChanges.getSaves());
|
||||
tm().delete(entityChanges.getDeletes());
|
||||
|
||||
@@ -19,7 +19,6 @@ import static google.registry.model.EppResourceUtils.isLinked;
|
||||
import static google.registry.model.EppResourceUtils.loadByForeignKey;
|
||||
import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
@@ -170,13 +169,13 @@ public final class ResourceFlowUtils {
|
||||
// The roid should match one of the contacts.
|
||||
Optional<VKey<ContactResource>> foundContact =
|
||||
domain.getReferencedContacts().stream()
|
||||
.filter(key -> key.getOfyKey().getName().equals(authRepoId))
|
||||
.filter(key -> key.getSqlKey().equals(authRepoId))
|
||||
.findFirst();
|
||||
if (!foundContact.isPresent()) {
|
||||
throw new BadAuthInfoForResourceException();
|
||||
}
|
||||
// Check the authInfo against the contact.
|
||||
verifyAuthInfo(authInfo, transactIfJpaTm(() -> tm().loadByKey(foundContact.get())));
|
||||
verifyAuthInfo(authInfo, tm().transact(() -> tm().loadByKey(foundContact.get())));
|
||||
}
|
||||
|
||||
/** Check that the given {@link AuthInfo} is valid for the given contact. */
|
||||
|
||||
@@ -17,7 +17,7 @@ package google.registry.flows;
|
||||
/**
|
||||
* Interface for a {@link Flow} that needs to be run transactionally.
|
||||
*
|
||||
* <p>Any flow that mutates Datastore should implement this so that {@link FlowRunner} will know how
|
||||
* to run it.
|
||||
* <p>Any flow that mutates the DB should implement this so that {@link FlowRunner} will know how to
|
||||
* run it.
|
||||
*/
|
||||
public interface TransactionalFlow extends Flow {}
|
||||
|
||||
@@ -26,6 +26,7 @@ import google.registry.flows.EppException.ParameterValuePolicyErrorException;
|
||||
import google.registry.flows.EppException.ParameterValueSyntaxErrorException;
|
||||
import google.registry.model.contact.ContactAddress;
|
||||
import google.registry.model.contact.ContactHistory;
|
||||
import google.registry.model.contact.ContactHistory.ContactHistoryId;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.contact.PostalInfo;
|
||||
import google.registry.model.poll.PendingActionNotificationResponse.ContactPendingActionNotificationResponse;
|
||||
@@ -84,7 +85,9 @@ public class ContactFlowUtils {
|
||||
transferData.getTransferStatus().isApproved(),
|
||||
transferData.getTransferRequestTrid(),
|
||||
now)))
|
||||
.setParentKey(contactHistoryKey)
|
||||
.setContactHistoryId(
|
||||
new ContactHistoryId(
|
||||
contactHistoryKey.getParent().getName(), contactHistoryKey.getId()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -96,7 +99,9 @@ public class ContactFlowUtils {
|
||||
.setEventTime(transferData.getPendingTransferExpirationTime())
|
||||
.setMsg(transferData.getTransferStatus().getMessage())
|
||||
.setResponseData(ImmutableList.of(createTransferResponse(targetId, transferData)))
|
||||
.setParentKey(contactHistoryKey)
|
||||
.setContactHistoryId(
|
||||
new ContactHistoryId(
|
||||
contactHistoryKey.getParent().getName(), contactHistoryKey.getId()))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -134,6 +134,8 @@ public final class ContactTransferRequestFlow implements TransactionalFlow {
|
||||
.asBuilder()
|
||||
.setTransferStatus(TransferStatus.PENDING)
|
||||
.setServerApproveEntities(
|
||||
serverApproveGainingPollMessage.getContactRepoId(),
|
||||
contactHistoryKey.getId(),
|
||||
ImmutableSet.of(
|
||||
serverApproveGainingPollMessage.createVKey(),
|
||||
serverApproveLosingPollMessage.createVKey()))
|
||||
|
||||
@@ -50,8 +50,7 @@ public class DomainPricingCustomLogic extends BaseFlowCustomLogic {
|
||||
|
||||
/** A hook that customizes the renew price. */
|
||||
@SuppressWarnings("unused")
|
||||
public FeesAndCredits customizeRenewPrice(RenewPriceParameters priceParameters)
|
||||
throws EppException {
|
||||
public FeesAndCredits customizeRenewPrice(RenewPriceParameters priceParameters) {
|
||||
return priceParameters.feesAndCredits();
|
||||
}
|
||||
|
||||
|
||||
@@ -87,6 +87,7 @@ import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainCommand;
|
||||
import google.registry.model.domain.DomainCommand.Create;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.domain.GracePeriod;
|
||||
import google.registry.model.domain.Period;
|
||||
import google.registry.model.domain.fee.FeeCreateCommandExtension;
|
||||
@@ -357,7 +358,8 @@ public final class DomainCreateFlow implements TransactionalFlow {
|
||||
.setIdnTableName(validateDomainNameWithIdnTables(domainName))
|
||||
.setRegistrationExpirationTime(registrationExpirationTime)
|
||||
.setAutorenewBillingEvent(autorenewBillingEvent.createVKey())
|
||||
.setAutorenewPollMessage(autorenewPollMessage.createVKey())
|
||||
.setAutorenewPollMessage(
|
||||
autorenewPollMessage.createVKey(), autorenewPollMessage.getHistoryRevisionId())
|
||||
.setLaunchNotice(hasClaimsNotice ? launchCreate.get().getNotice() : null)
|
||||
.setSmdId(signedMarkId)
|
||||
.setDsData(secDnsCreate.map(SecDnsCreateExtension::getDsData).orElse(null))
|
||||
@@ -576,7 +578,8 @@ public final class DomainCreateFlow implements TransactionalFlow {
|
||||
.setRegistrarId(registrarId)
|
||||
.setEventTime(registrationExpirationTime)
|
||||
.setMsg("Domain was auto-renewed.")
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -608,7 +611,7 @@ public final class DomainCreateFlow implements TransactionalFlow {
|
||||
ImmutableList.of(
|
||||
DomainPendingActionNotificationResponse.create(
|
||||
fullyQualifiedDomainName, true, historyEntry.getTrid(), now)))
|
||||
.setParent(historyEntry)
|
||||
.setHistoryEntry(historyEntry)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.domain.GracePeriod;
|
||||
import google.registry.model.domain.fee.BaseFee.FeeType;
|
||||
import google.registry.model.domain.fee.Credit;
|
||||
@@ -355,7 +356,8 @@ public final class DomainDeleteFlow implements TransactionalFlow {
|
||||
ImmutableList.of(
|
||||
DomainPendingActionNotificationResponse.create(
|
||||
existingDomain.getDomainName(), true, trid, deletionTime)))
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -367,7 +369,8 @@ public final class DomainDeleteFlow implements TransactionalFlow {
|
||||
return new PollMessage.OneTime.Builder()
|
||||
.setRegistrarId(existingDomain.getPersistedCurrentSponsorRegistrarId())
|
||||
.setEventTime(now)
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.setMsg(
|
||||
String.format(
|
||||
"Domain %s was deleted by registry administrator with final deletion effective: %s",
|
||||
|
||||
@@ -63,7 +63,6 @@ import com.google.common.collect.Multimaps;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.net.InternetDomainName;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.flows.EppException;
|
||||
import google.registry.flows.EppException.AuthorizationErrorException;
|
||||
import google.registry.flows.EppException.CommandUseErrorException;
|
||||
@@ -89,6 +88,7 @@ import google.registry.model.domain.DomainCommand.CreateOrUpdate;
|
||||
import google.registry.model.domain.DomainCommand.InvalidReferencesException;
|
||||
import google.registry.model.domain.DomainCommand.Update;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.domain.ForeignKeyedDesignatedContact;
|
||||
import google.registry.model.domain.Period;
|
||||
import google.registry.model.domain.fee.BaseFee;
|
||||
@@ -592,14 +592,15 @@ public class DomainFlowUtils {
|
||||
// where all autorenew poll messages had already been delivered (this would cause the poll
|
||||
// message to be deleted), and then subsequently the transfer was canceled, rejected, or deleted
|
||||
// (which would cause the poll message to be recreated here).
|
||||
Key<PollMessage.Autorenew> existingAutorenewKey = domain.getAutorenewPollMessage().getOfyKey();
|
||||
PollMessage.Autorenew updatedAutorenewPollMessage =
|
||||
autorenewPollMessage.isPresent()
|
||||
? autorenewPollMessage.get().asBuilder().setAutorenewEndTime(newEndTime).build()
|
||||
: newAutorenewPollMessage(domain)
|
||||
.setId(existingAutorenewKey.getId())
|
||||
.setId((Long) domain.getAutorenewPollMessage().getSqlKey())
|
||||
.setAutorenewEndTime(newEndTime)
|
||||
.setParentKey(existingAutorenewKey.getParent())
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(
|
||||
domain.getRepoId(), domain.getAutorenewPollMessageHistoryId()))
|
||||
.build();
|
||||
|
||||
// If the resultant autorenew poll message would have no poll messages to deliver, then just
|
||||
|
||||
@@ -22,7 +22,6 @@ import static google.registry.flows.domain.DomainFlowUtils.handleFeeRequest;
|
||||
import static google.registry.flows.domain.DomainFlowUtils.loadForeignKeyedDesignatedContacts;
|
||||
import static google.registry.model.EppResourceUtils.loadByForeignKey;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
@@ -110,7 +109,7 @@ public final class DomainInfoFlow implements Flow {
|
||||
.setRepoId(domain.getRepoId())
|
||||
.setCurrentSponsorClientId(domain.getCurrentSponsorRegistrarId())
|
||||
.setRegistrant(
|
||||
transactIfJpaTm(() -> tm().loadByKey(domain.getRegistrant())).getContactId());
|
||||
tm().transact(() -> tm().loadByKey(domain.getRegistrant())).getContactId());
|
||||
// If authInfo is non-null, then the caller is authorized to see the full information since we
|
||||
// will have already verified the authInfo is valid.
|
||||
if (registrarId.equals(domain.getCurrentSponsorRegistrarId()) || authInfo.isPresent()) {
|
||||
@@ -118,7 +117,7 @@ public final class DomainInfoFlow implements Flow {
|
||||
infoBuilder
|
||||
.setStatusValues(domain.getStatusValues())
|
||||
.setContacts(
|
||||
transactIfJpaTm(() -> loadForeignKeyedDesignatedContacts(domain.getContacts())))
|
||||
tm().transact(() -> loadForeignKeyedDesignatedContacts(domain.getContacts())))
|
||||
.setNameservers(hostsRequest.requestDelegated() ? domain.loadNameserverHostNames() : null)
|
||||
.setSubordinateHosts(
|
||||
hostsRequest.requestSubordinate() ? domain.getSubordinateHosts() : null)
|
||||
|
||||
@@ -51,10 +51,12 @@ import org.joda.time.DateTime;
|
||||
*/
|
||||
public final class DomainPricingLogic {
|
||||
|
||||
@Inject DomainPricingCustomLogic customLogic;
|
||||
private final DomainPricingCustomLogic customLogic;
|
||||
|
||||
@Inject
|
||||
DomainPricingLogic() {}
|
||||
public DomainPricingLogic(DomainPricingCustomLogic customLogic) {
|
||||
this.customLogic = customLogic;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new create price for the pricer.
|
||||
@@ -105,13 +107,12 @@ public final class DomainPricingLogic {
|
||||
}
|
||||
|
||||
/** Returns a new renewal cost for the pricer. */
|
||||
FeesAndCredits getRenewPrice(
|
||||
public FeesAndCredits getRenewPrice(
|
||||
Registry registry,
|
||||
String domainName,
|
||||
DateTime dateTime,
|
||||
int years,
|
||||
@Nullable Recurring recurringBillingEvent)
|
||||
throws EppException {
|
||||
@Nullable Recurring recurringBillingEvent) {
|
||||
checkArgument(years > 0, "Number of years must be positive");
|
||||
Money renewCost;
|
||||
boolean isRenewCostPremiumPrice;
|
||||
|
||||
@@ -54,9 +54,11 @@ import google.registry.flows.custom.EntityChanges;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.billing.BillingEvent.OneTime;
|
||||
import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.billing.BillingEvent.Recurring;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainCommand.Renew;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.domain.DomainRenewData;
|
||||
import google.registry.model.domain.GracePeriod;
|
||||
import google.registry.model.domain.Period;
|
||||
@@ -153,9 +155,15 @@ public final class DomainRenewFlow implements TransactionalFlow {
|
||||
validateRegistrationPeriod(now, newExpirationTime);
|
||||
Optional<FeeRenewCommandExtension> feeRenew =
|
||||
eppInput.getSingleExtension(FeeRenewCommandExtension.class);
|
||||
Recurring existingRecurringBillingEvent =
|
||||
tm().loadByKey(existingDomain.getAutorenewBillingEvent());
|
||||
FeesAndCredits feesAndCredits =
|
||||
pricingLogic.getRenewPrice(
|
||||
Registry.get(existingDomain.getTld()), targetId, now, years, null);
|
||||
Registry.get(existingDomain.getTld()),
|
||||
targetId,
|
||||
now,
|
||||
years,
|
||||
existingRecurringBillingEvent);
|
||||
validateFeeChallenge(targetId, now, feeRenew, feesAndCredits);
|
||||
flowCustomLogic.afterValidation(
|
||||
AfterValidationParameters.newBuilder()
|
||||
@@ -173,12 +181,16 @@ public final class DomainRenewFlow implements TransactionalFlow {
|
||||
BillingEvent.Recurring newAutorenewEvent =
|
||||
newAutorenewBillingEvent(existingDomain)
|
||||
.setEventTime(newExpirationTime)
|
||||
.setRenewalPrice(existingRecurringBillingEvent.getRenewalPrice().orElse(null))
|
||||
.setRenewalPriceBehavior(existingRecurringBillingEvent.getRenewalPriceBehavior())
|
||||
.setParent(domainHistoryKey)
|
||||
.build();
|
||||
PollMessage.Autorenew newAutorenewPollMessage =
|
||||
newAutorenewPollMessage(existingDomain)
|
||||
.setEventTime(newExpirationTime)
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(
|
||||
domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.build();
|
||||
// End the old autorenew billing event and poll message now. This may delete the poll message.
|
||||
updateAutorenewRecurrenceEndTime(existingDomain, now);
|
||||
@@ -189,7 +201,9 @@ public final class DomainRenewFlow implements TransactionalFlow {
|
||||
.setLastEppUpdateRegistrarId(registrarId)
|
||||
.setRegistrationExpirationTime(newExpirationTime)
|
||||
.setAutorenewBillingEvent(newAutorenewEvent.createVKey())
|
||||
.setAutorenewPollMessage(newAutorenewPollMessage.createVKey())
|
||||
.setAutorenewPollMessage(
|
||||
newAutorenewPollMessage.createVKey(),
|
||||
newAutorenewPollMessage.getHistoryRevisionId())
|
||||
.addGracePeriod(
|
||||
GracePeriod.forBillingEvent(
|
||||
GracePeriodStatus.RENEW, existingDomain.getRepoId(), explicitRenewEvent))
|
||||
|
||||
@@ -53,6 +53,7 @@ import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainCommand.Update;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.domain.fee.BaseFee.FeeType;
|
||||
import google.registry.model.domain.fee.Fee;
|
||||
import google.registry.model.domain.fee.FeeTransformResponseExtension;
|
||||
@@ -173,7 +174,9 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow {
|
||||
newAutorenewPollMessage(existingDomain)
|
||||
.setEventTime(newExpirationTime)
|
||||
.setAutorenewEndTime(END_OF_TIME)
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(
|
||||
domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.build();
|
||||
DomainBase newDomain =
|
||||
performRestore(
|
||||
@@ -245,7 +248,8 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow {
|
||||
.setGracePeriods(null)
|
||||
.setDeletePollMessage(null)
|
||||
.setAutorenewBillingEvent(autorenewEvent.createVKey())
|
||||
.setAutorenewPollMessage(autorenewPollMessage.createVKey())
|
||||
.setAutorenewPollMessage(
|
||||
autorenewPollMessage.createVKey(), autorenewPollMessage.getHistoryRevisionId())
|
||||
// Clear the autorenew end time so if it had expired but is now explicitly being restored,
|
||||
// it won't immediately be deleted again.
|
||||
.setAutorenewEndTime(Optional.empty())
|
||||
|
||||
@@ -51,6 +51,7 @@ import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.domain.GracePeriod;
|
||||
import google.registry.model.domain.metadata.MetadataExtension;
|
||||
import google.registry.model.domain.rgp.GracePeriodStatus;
|
||||
@@ -176,7 +177,9 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
|
||||
.setEventTime(newExpirationTime)
|
||||
.setAutorenewEndTime(END_OF_TIME)
|
||||
.setMsg("Domain was auto-renewed.")
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(
|
||||
domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.build();
|
||||
// Construct the post-transfer domain.
|
||||
DomainBase partiallyApprovedDomain =
|
||||
@@ -194,7 +197,9 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
|
||||
.build())
|
||||
.setRegistrationExpirationTime(newExpirationTime)
|
||||
.setAutorenewBillingEvent(autorenewEvent.createVKey())
|
||||
.setAutorenewPollMessage(gainingClientAutorenewPollMessage.createVKey())
|
||||
.setAutorenewPollMessage(
|
||||
gainingClientAutorenewPollMessage.createVKey(),
|
||||
gainingClientAutorenewPollMessage.getHistoryRevisionId())
|
||||
// Remove all the old grace periods and add a new one for the transfer.
|
||||
.setGracePeriods(
|
||||
billingEvent
|
||||
|
||||
@@ -63,7 +63,7 @@ import org.joda.time.DateTime;
|
||||
* transfer is automatically approved. Within that window, this flow allows the losing client to
|
||||
* reject the transfer request.
|
||||
*
|
||||
* <p>When the transfer was requested, poll messages and billing events were saved to Datastore with
|
||||
* <p>When the transfer was requested, poll messages and billing events were saved to SQL with
|
||||
* timestamps such that they only would become active when the transfer period passed. In this flow,
|
||||
* those speculative objects are deleted.
|
||||
*
|
||||
|
||||
@@ -208,6 +208,8 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
|
||||
// Create the transfer data that represents the pending transfer.
|
||||
DomainTransferData pendingTransferData =
|
||||
createPendingTransferData(
|
||||
domainAtTransferTime.getRepoId(),
|
||||
domainHistoryKey.getId(),
|
||||
new DomainTransferData.Builder()
|
||||
.setTransferRequestTrid(trid)
|
||||
.setTransferRequestTime(now)
|
||||
|
||||
@@ -26,6 +26,7 @@ import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.domain.DomainHistory.DomainHistoryId;
|
||||
import google.registry.model.domain.GracePeriod;
|
||||
import google.registry.model.domain.Period;
|
||||
import google.registry.model.domain.rgp.GracePeriodStatus;
|
||||
@@ -51,6 +52,8 @@ public final class DomainTransferUtils {
|
||||
|
||||
/** Sets up {@link TransferData} for a domain with links to entities for server approval. */
|
||||
public static DomainTransferData createPendingTransferData(
|
||||
String domainRepoId,
|
||||
Long historyId,
|
||||
DomainTransferData.Builder transferDataBuilder,
|
||||
ImmutableSet<TransferServerApproveEntity> serverApproveEntities,
|
||||
Period transferPeriod) {
|
||||
@@ -82,7 +85,7 @@ public final class DomainTransferUtils {
|
||||
.map(PollMessage.Autorenew.class::cast)
|
||||
.collect(onlyElement())
|
||||
.createVKey())
|
||||
.setServerApproveEntities(serverApproveEntityKeys.build())
|
||||
.setServerApproveEntities(domainRepoId, historyId, serverApproveEntityKeys.build())
|
||||
.setTransferPeriod(transferPeriod)
|
||||
.build();
|
||||
}
|
||||
@@ -180,7 +183,8 @@ public final class DomainTransferUtils {
|
||||
transferData.getTransferStatus().isApproved(),
|
||||
transferData.getTransferRequestTrid(),
|
||||
now)))
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -197,7 +201,8 @@ public final class DomainTransferUtils {
|
||||
.setResponseData(
|
||||
ImmutableList.of(
|
||||
createTransferResponse(targetId, transferData, extendedRegistrationExpirationTime)))
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -228,7 +233,8 @@ public final class DomainTransferUtils {
|
||||
.setEventTime(serverApproveNewExpirationTime)
|
||||
.setAutorenewEndTime(END_OF_TIME)
|
||||
.setMsg("Domain was auto-renewed.")
|
||||
.setParentKey(domainHistoryKey)
|
||||
.setDomainHistoryId(
|
||||
new DomainHistoryId(domainHistoryKey.getParent().getName(), domainHistoryKey.getId()))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,6 @@ import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedSet;
|
||||
import com.google.common.collect.Ordering;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.InternetDomainName;
|
||||
import google.registry.dns.DnsQueue;
|
||||
import google.registry.flows.EppException;
|
||||
@@ -67,6 +66,7 @@ import google.registry.flows.domain.DomainFlowUtils.NameserversNotSpecifiedForTl
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.domain.DesignatedContact;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainCommand.Update;
|
||||
import google.registry.model.domain.DomainCommand.Update.AddRemove;
|
||||
@@ -146,8 +146,6 @@ public final class DomainUpdateFlow implements TransactionalFlow {
|
||||
private static final ImmutableSet<StatusValue> UPDATE_DISALLOWED_STATUSES =
|
||||
ImmutableSet.of(StatusValue.PENDING_DELETE, StatusValue.SERVER_UPDATE_PROHIBITED);
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject ResourceCommand resourceCommand;
|
||||
@Inject ExtensionManager extensionManager;
|
||||
@Inject EppInput eppInput;
|
||||
@@ -246,6 +244,13 @@ public final class DomainUpdateFlow implements TransactionalFlow {
|
||||
validateRegistrantIsntBeingRemoved(change);
|
||||
Optional<SecDnsUpdateExtension> secDnsUpdate =
|
||||
eppInput.getSingleExtension(SecDnsUpdateExtension.class);
|
||||
|
||||
// We have to verify no duplicate contacts _before_ constructing the domain because it is
|
||||
// illegal to construct a domain with duplicate contacts.
|
||||
Sets.SetView<DesignatedContact> newContacts =
|
||||
Sets.union(Sets.difference(domain.getContacts(), remove.getContacts()), add.getContacts());
|
||||
validateNoDuplicateContacts(newContacts);
|
||||
|
||||
DomainBase.Builder domainBuilder =
|
||||
domain
|
||||
.asBuilder()
|
||||
@@ -266,8 +271,8 @@ public final class DomainUpdateFlow implements TransactionalFlow {
|
||||
.removeStatusValues(remove.getStatusValues())
|
||||
.addNameservers(add.getNameservers().stream().collect(toImmutableSet()))
|
||||
.removeNameservers(remove.getNameservers().stream().collect(toImmutableSet()))
|
||||
.addContacts(add.getContacts())
|
||||
.removeContacts(remove.getContacts())
|
||||
.addContacts(add.getContacts())
|
||||
.setRegistrant(firstNonNull(change.getRegistrant(), domain.getRegistrant()))
|
||||
.setAuthInfo(firstNonNull(change.getAuthInfo(), domain.getAuthInfo()));
|
||||
Optional<DomainUpdateSuperuserExtension> superuserExt =
|
||||
@@ -289,7 +294,6 @@ public final class DomainUpdateFlow implements TransactionalFlow {
|
||||
}
|
||||
|
||||
private void validateNewState(DomainBase newDomain) throws EppException {
|
||||
validateNoDuplicateContacts(newDomain.getContacts());
|
||||
validateRequiredContactsPresent(newDomain.getRegistrant(), newDomain.getContacts());
|
||||
validateDsData(newDomain.getDsData());
|
||||
validateNameserversCountForTld(
|
||||
@@ -363,7 +367,7 @@ public final class DomainUpdateFlow implements TransactionalFlow {
|
||||
|
||||
return Optional.ofNullable(
|
||||
new PollMessage.OneTime.Builder()
|
||||
.setParent(historyEntry)
|
||||
.setHistoryEntry(historyEntry)
|
||||
.setEventTime(now)
|
||||
.setRegistrarId(existingDomain.getCurrentSponsorRegistrarId())
|
||||
.setMsg(msg)
|
||||
|
||||
@@ -16,7 +16,6 @@ package google.registry.flows.domain.token;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
@@ -161,7 +160,7 @@ public class AllocationTokenFlowUtils {
|
||||
throw new InvalidAllocationTokenException();
|
||||
}
|
||||
Optional<AllocationToken> maybeTokenEntity =
|
||||
transactIfJpaTm(() -> tm().loadByKeyIfPresent(VKey.create(AllocationToken.class, token)));
|
||||
tm().transact(() -> tm().loadByKeyIfPresent(VKey.create(AllocationToken.class, token)));
|
||||
if (!maybeTokenEntity.isPresent()) {
|
||||
throw new InvalidAllocationTokenException();
|
||||
}
|
||||
|
||||
@@ -72,8 +72,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
|
||||
StatusValue.PENDING_DELETE,
|
||||
StatusValue.SERVER_DELETE_PROHIBITED);
|
||||
|
||||
private static final DnsQueue dnsQueue = DnsQueue.create();
|
||||
|
||||
@Inject DnsQueue dnsQueue;
|
||||
@Inject ExtensionManager extensionManager;
|
||||
@Inject @RegistrarId String registrarId;
|
||||
@Inject @TargetId String targetId;
|
||||
|
||||
@@ -19,7 +19,6 @@ import static google.registry.flows.ResourceFlowUtils.loadAndVerifyExistence;
|
||||
import static google.registry.flows.host.HostFlowUtils.validateHostName;
|
||||
import static google.registry.model.EppResourceUtils.isLinked;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.flows.EppException;
|
||||
@@ -78,8 +77,8 @@ public final class HostInfoFlow implements Flow {
|
||||
// there is no superordinate domain, the host's own values for these fields will be correct.
|
||||
if (host.isSubordinate()) {
|
||||
DomainBase superordinateDomain =
|
||||
transactIfJpaTm(
|
||||
() -> tm().loadByKey(host.getSuperordinateDomain()).cloneProjectedAtTime(now));
|
||||
tm().transact(
|
||||
() -> tm().loadByKey(host.getSuperordinateDomain()).cloneProjectedAtTime(now));
|
||||
hostInfoDataBuilder
|
||||
.setCurrentSponsorClientId(superordinateDomain.getCurrentSponsorRegistrarId())
|
||||
.setLastTransferTime(host.computeLastTransferTime(superordinateDomain));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user