1
0
mirror of https://github.com/google/nomulus synced 2026-07-06 08:06:33 +00:00

Compare commits

...

7 Commits

Author SHA1 Message Date
sarahcaseybot acdbc65c51 Change Registry object reference to Tld in configuration.md (#2021) 2023-05-12 12:32:02 -04:00
Weimin Yu d510531f65 Remove the deprecatd DefaultCredential (#2032)
Use the ApplicationDefaultCredential annotation instead.

The new annotation has been verified in sandbox and production using the
'executeCannedScript' endpoint. The verification code is removed in this
PR too.
2023-05-11 13:46:36 -04:00
Lai Jiang 0d4dd57fe7 Fix a typo (#2031) 2023-05-11 13:26:07 -04:00
Pavlo Tkach 2667a0e977 Expand nomulus get_domain command to load up deleted domain data too (#2018) 2023-05-10 16:05:03 -04:00
gbrodman 1aef31efff Allow usage of standard HTTP requests in CloudTasksUtils (#2013)
This adds a possible configuration point "defaultServiceAccount" (which
in GAE will be the standard GAE service account). If this is configured,
CloudTasksUtils can create tasks with standard HTTP requests with an
OIDC token corresponding to that service account, as opposed to using
the AppEngine-specific request methods.

This also works with IAP, in that if IAP is on and we specify the IAP
client ID in the config, CloudTasksUtils will use the IAP client ID as
the token audience and the request will successfully be passed through
the IAP layer.

Tetsted in QA.
2023-05-09 16:02:12 -04:00
Lai Jiang 4d19245c29 Change usage grouping key in the invoice CSV (#2024)
This column is used by the billing team to create invoices. Registrars
have asked that a single invoice be created for a given registrar,
instead of one per registrar-tld pair. This should have no other effect
on the billing pipeline as the invoice grouping key has a description
field that also contains the TLD, so the granularity as a whole does not
change.
2023-05-09 11:25:11 -04:00
Lai Jiang 4b34307a6e Delete DatabaseMigrationStateSchedule (#2001)
We have been using it as a poor man's timed flag that triggers a system
behavior change after a certain time. We have no foreseeable future use
for it now that the DNS pull queue related code is deleted. If in the
future a need for such a flag arises, we are better off implementing a
proper flag system than hijacking this class any way.
2023-05-08 14:36:28 -04:00
38 changed files with 470 additions and 1332 deletions
@@ -17,7 +17,6 @@ package google.registry.batch;
import static google.registry.request.Action.Method.POST;
import com.google.common.flogger.FluentLogger;
import google.registry.batch.cannedscript.CannedScripts;
import google.registry.request.Action;
import google.registry.request.auth.Auth;
import javax.inject.Inject;
@@ -25,15 +24,15 @@ import javax.inject.Inject;
/**
* Action that executes a canned script specified by the caller.
*
* <p>This class is introduced to help the safe rollout of credential changes. The delegated
* credentials in particular, benefit from this: they require manual configuration of the peer
* system in each environment, and may wait hours or even days after deployment until triggered by
* user activities.
* <p>This class provides a hook for invoking hard-coded methods. The main use case is to verify in
* Sandbox and Production environments new features that depend on environment-specific
* configurations. For example, the {@code DelegatedCredential}, which requires correct GWorkspace
* configuration, has been tested this way. Since it is a hassle to add or remove endpoints, we keep
* this class all the time.
*
* <p>This action can be invoked using the Nomulus CLI command: {@code nomulus -e ${env} curl
* --service BACKEND -X POST -u '/_dr/task/executeCannedScript?script=${script_name}'}
* --service BACKEND -X POST -u '/_dr/task/executeCannedScript}'}
*/
// TODO(b/277239043): remove class after credential changes are rolled out.
@Action(
service = Action.Service.BACKEND,
path = "/_dr/task/executeCannedScript",
@@ -51,7 +50,7 @@ public class CannedScriptExecutionAction implements Runnable {
@Override
public void run() {
try {
CannedScripts.runAllChecks();
// Invoke canned scripts here.
logger.atInfo().log("Finished running scripts.");
} catch (Throwable t) {
logger.atWarning().withCause(t).log("Error executing scripts.");
@@ -16,6 +16,7 @@ package google.registry.batch;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.tools.ServiceConnection.getServer;
import static java.util.concurrent.TimeUnit.SECONDS;
import com.google.api.gax.rpc.ApiException;
@@ -23,6 +24,8 @@ import com.google.cloud.tasks.v2.AppEngineHttpRequest;
import com.google.cloud.tasks.v2.AppEngineRouting;
import com.google.cloud.tasks.v2.CloudTasksClient;
import com.google.cloud.tasks.v2.HttpMethod;
import com.google.cloud.tasks.v2.HttpRequest;
import com.google.cloud.tasks.v2.OidcToken;
import com.google.cloud.tasks.v2.QueueName;
import com.google.cloud.tasks.v2.Task;
import com.google.common.base.Joiner;
@@ -46,7 +49,10 @@ import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Optional;
import java.util.Random;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Supplier;
import javax.annotation.Nullable;
import javax.inject.Inject;
import org.joda.time.Duration;
@@ -61,6 +67,9 @@ public class CloudTasksUtils implements Serializable {
private final Clock clock;
private final String projectId;
private final String locationId;
// defaultServiceAccount and iapClientId are nullable because Optional isn't serializable
@Nullable private final String defaultServiceAccount;
@Nullable private final String iapClientId;
private final SerializableCloudTasksClient client;
@Inject
@@ -69,11 +78,15 @@ public class CloudTasksUtils implements Serializable {
Clock clock,
@Config("projectId") String projectId,
@Config("locationId") String locationId,
@Config("defaultServiceAccount") Optional<String> defaultServiceAccount,
@Config("iapClientId") Optional<String> iapClientId,
SerializableCloudTasksClient client) {
this.retrier = retrier;
this.clock = clock;
this.projectId = projectId;
this.locationId = locationId;
this.defaultServiceAccount = defaultServiceAccount.orElse(null);
this.iapClientId = iapClientId.orElse(null);
this.client = client;
}
@@ -98,6 +111,74 @@ public class CloudTasksUtils implements Serializable {
return enqueue(queue, Arrays.asList(tasks));
}
/**
* Converts a (possible) set of params into an HTTP request via the appropriate method.
*
* <p>For GET requests we add them on to the URL, and for POST requests we add them in the body of
* the request.
*
* <p>The parameters {@code putHeadersFunction} and {@code setBodyFunction} are used so that this
* method can be called with either an AppEngine HTTP request or a standard non-AppEngine HTTP
* request. The two objects do not have the same methods, but both have ways of setting headers /
* body.
*
* @return the resulting path (unchanged for POST requests, with params added for GET requests)
*/
private String processRequestParameters(
String path,
HttpMethod method,
Multimap<String, String> params,
BiConsumer<String, String> putHeadersFunction,
Consumer<ByteString> setBodyFunction) {
if (CollectionUtils.isNullOrEmpty(params)) {
return path;
}
Escaper escaper = UrlEscapers.urlPathSegmentEscaper();
String encodedParams =
Joiner.on("&")
.join(
params.entries().stream()
.map(
entry ->
String.format(
"%s=%s",
escaper.escape(entry.getKey()), escaper.escape(entry.getValue())))
.collect(toImmutableList()));
if (method.equals(HttpMethod.GET)) {
return String.format("%s?%s", path, encodedParams);
}
putHeadersFunction.accept(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString());
setBodyFunction.accept(ByteString.copyFrom(encodedParams, StandardCharsets.UTF_8));
return path;
}
/**
* Creates a {@link Task} that does not use AppEngine for submission.
*
* <p>This uses the standard Cloud Tasks auth format to create and send an OIDC ID token set to
* the default service account. That account must have permission to submit tasks to Cloud Tasks.
*/
private Task createNonAppEngineTask(
String path, HttpMethod method, Service service, Multimap<String, String> params) {
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder().setHttpMethod(method);
path =
processRequestParameters(
path, method, params, requestBuilder::putHeaders, requestBuilder::setBody);
OidcToken.Builder oidcTokenBuilder =
OidcToken.newBuilder().setServiceAccountEmail(defaultServiceAccount);
// If the service is using IAP, add that as the audience for the token so the request can be
// appropriately authed. Otherwise, use the project name.
if (iapClientId != null) {
oidcTokenBuilder.setAudience(iapClientId);
} else {
oidcTokenBuilder.setAudience(projectId);
}
requestBuilder.setOidcToken(oidcTokenBuilder.build());
String totalPath = String.format("%s%s", getServer(service), path);
requestBuilder.setUrl(totalPath);
return Task.newBuilder().setHttpRequest(requestBuilder.build()).build();
}
/**
* Create a {@link Task} to be enqueued.
*
@@ -123,34 +204,21 @@ public class CloudTasksUtils implements Serializable {
method.equals(HttpMethod.GET) || method.equals(HttpMethod.POST),
"HTTP method %s is used. Only GET and POST are allowed.",
method);
AppEngineHttpRequest.Builder requestBuilder =
AppEngineHttpRequest.newBuilder()
.setHttpMethod(method)
.setAppEngineRouting(
AppEngineRouting.newBuilder().setService(service.toString()).build());
if (!CollectionUtils.isNullOrEmpty(params)) {
Escaper escaper = UrlEscapers.urlPathSegmentEscaper();
String encodedParams =
Joiner.on("&")
.join(
params.entries().stream()
.map(
entry ->
String.format(
"%s=%s",
escaper.escape(entry.getKey()), escaper.escape(entry.getValue())))
.collect(toImmutableList()));
if (method == HttpMethod.GET) {
path = String.format("%s?%s", path, encodedParams);
} else {
requestBuilder
.putHeaders(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
.setBody(ByteString.copyFrom(encodedParams, StandardCharsets.UTF_8));
}
// If the default service account is configured, send a standard non-AppEngine HTTP request
if (defaultServiceAccount != null) {
return createNonAppEngineTask(path, method, service, params);
} else {
AppEngineHttpRequest.Builder requestBuilder =
AppEngineHttpRequest.newBuilder()
.setHttpMethod(method)
.setAppEngineRouting(
AppEngineRouting.newBuilder().setService(service.toString()).build());
path =
processRequestParameters(
path, method, params, requestBuilder::putHeaders, requestBuilder::setBody);
requestBuilder.setRelativeUri(path);
return Task.newBuilder().setAppEngineHttpRequest(requestBuilder.build()).build();
}
requestBuilder.setRelativeUri(path);
return Task.newBuilder().setAppEngineHttpRequest(requestBuilder.build()).build();
}
/**
@@ -1,199 +0,0 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.batch.cannedscript;
import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.dataflow.Dataflow;
import com.google.api.services.dns.Dns;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import com.google.cloud.tasks.v2.CloudTasksClient;
import com.google.cloud.tasks.v2.CloudTasksSettings;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.flogger.FluentLogger;
import dagger.Component;
import dagger.Module;
import dagger.Provides;
import google.registry.config.CredentialModule;
import google.registry.config.CredentialModule.ApplicationDefaultCredential;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.util.GoogleCredentialsBundle;
import google.registry.util.UtilsModule;
import java.io.IOException;
import java.util.Optional;
import javax.inject.Singleton;
/** Canned actions invoked from {@link google.registry.batch.CannedScriptExecutionAction}. */
// TODO(b/277239043): remove class after credential changes are rolled out.
public class CannedScripts {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final Supplier<CannedScriptsComponent> COMPONENT_SUPPLIER =
Suppliers.memoize(DaggerCannedScripts_CannedScriptsComponent::create);
public static void runAllChecks() {
CannedScriptsComponent component = COMPONENT_SUPPLIER.get();
String projectId = component.projectId();
Bigquery bigquery = component.bigQuery();
try {
bigquery.datasets().list(projectId).execute().getDatasets().stream()
.findAny()
.ifPresent(
datasets ->
logger.atInfo().log("Found a BQ dataset [%s]", datasets.getFriendlyName()));
logger.atInfo().log("Finished accessing BQ.");
} catch (IOException ioe) {
logger.atSevere().withCause(ioe).log("Failed to access bigquery.");
}
try {
Dataflow dataflow = component.dataflow();
dataflow.projects().jobs().list(projectId).execute().getJobs().stream()
.findAny()
.ifPresent(job -> logger.atInfo().log("Found a job [%s]", job.getName()));
logger.atInfo().log("Finished accessing Dataflow.");
} catch (IOException ioe) {
logger.atSevere().withCause(ioe).log("Failed to access dataflow.");
}
try {
Storage gcs = component.gcs();
gcs.listAcls(projectId + "-beam");
logger.atInfo().log("Finished accessing gcs.");
} catch (RuntimeException e) {
logger.atSevere().withCause(e).log("Failed to access gcs.");
}
try {
Dns dns = component.dns();
dns.managedZones().list(projectId).execute().getManagedZones().stream()
.findAny()
.ifPresent(zone -> logger.atInfo().log("Found one zone [%s].", zone.getName()));
logger.atInfo().log("Finished accessing dns.");
} catch (IOException ioe) {
logger.atSevere().withCause(ioe).log("Failed to access dns.");
}
try {
CloudTasksClient client = component.cloudtasksClient();
com.google.cloud.tasks.v2.Queue queue =
client.getQueue(
String.format(
"projects/%s/locations/%s/queues/async-actions",
projectId, component.locationId()));
logger.atInfo().log("Got async queue state [%s]", queue.getState().name());
logger.atInfo().log("Finished accessing cloudtasks.");
} catch (RuntimeException e) {
logger.atSevere().withCause(e).log("Failed to access cloudtasks.");
}
}
@Singleton
@Component(
modules = {
ConfigModule.class,
CredentialModule.class,
CannedScriptsModule.class,
UtilsModule.class
})
interface CannedScriptsComponent {
Bigquery bigQuery();
CloudTasksClient cloudtasksClient();
Dataflow dataflow();
Dns dns();
Storage gcs();
@Config("projectId")
String projectId();
@Config("locationId")
String locationId();
}
@Module
static class CannedScriptsModule {
@Provides
static Bigquery provideBigquery(
@ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
@Config("projectId") String projectId) {
return new Bigquery.Builder(
credentialsBundle.getHttpTransport(),
credentialsBundle.getJsonFactory(),
credentialsBundle.getHttpRequestInitializer())
.setApplicationName(projectId)
.build();
}
@Provides
static Dataflow provideDataflow(
@ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
@Config("projectId") String projectId) {
return new Dataflow.Builder(
credentialsBundle.getHttpTransport(),
credentialsBundle.getJsonFactory(),
credentialsBundle.getHttpRequestInitializer())
.setApplicationName(String.format("%s billing", projectId))
.build();
}
@Provides
static Storage provideGcs(
@ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle) {
return StorageOptions.newBuilder()
.setCredentials(credentialsBundle.getGoogleCredentials())
.build()
.getService();
}
@Provides
static Dns provideDns(
@ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
@Config("projectId") String projectId,
@Config("cloudDnsRootUrl") Optional<String> rootUrl,
@Config("cloudDnsServicePath") Optional<String> servicePath) {
Dns.Builder builder =
new Dns.Builder(
credentialsBundle.getHttpTransport(),
credentialsBundle.getJsonFactory(),
credentialsBundle.getHttpRequestInitializer())
.setApplicationName(projectId);
rootUrl.ifPresent(builder::setRootUrl);
servicePath.ifPresent(builder::setServicePath);
return builder.build();
}
@Provides
public static CloudTasksClient provideCloudTasksClient(
@ApplicationDefaultCredential GoogleCredentialsBundle credentials) {
CloudTasksClient client;
try {
client =
CloudTasksClient.create(
CloudTasksSettings.newBuilder()
.setCredentialsProvider(
FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
.build());
} catch (IOException e) {
throw new RuntimeException(e);
}
return client;
}
}
}
@@ -189,7 +189,7 @@ public abstract class BillingEvent implements Serializable {
.minusDays(1)
.toString(),
billingId(),
String.format("%s - %s", registrarId(), tld()),
registrarId(),
String.format("%s | TLD: %s | TERM: %d-year", action(), tld(), years()),
amount(),
currency(),
@@ -233,7 +233,7 @@ public abstract class BillingEvent implements Serializable {
/** Returns the billing account id, which is the {@code BillingEvent.billingId}. */
abstract String productAccountKey();
/** Returns the invoice grouping key, which is in the format "registrarId - tld". */
/** Returns the invoice grouping key, which is the registrar ID. */
abstract String usageGroupingKey();
/** Returns a description of the item, formatted as "action | TLD: tld | TERM: n-year." */
@@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList;
import dagger.Module;
import dagger.Provides;
import dagger.multibindings.Multibinds;
import google.registry.config.CredentialModule.DefaultCredential;
import google.registry.config.CredentialModule.ApplicationDefaultCredential;
import google.registry.config.RegistryConfig.Config;
import google.registry.util.GoogleCredentialsBundle;
import java.util.Map;
@@ -34,7 +34,7 @@ public abstract class BigqueryModule {
@Provides
static Bigquery provideBigquery(
@DefaultCredential GoogleCredentialsBundle credentialsBundle,
@ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
@Config("projectId") String projectId) {
return new Bigquery.Builder(
credentialsBundle.getHttpTransport(),
@@ -22,7 +22,7 @@ import dagger.Provides;
import google.registry.batch.CloudTasksUtils;
import google.registry.batch.CloudTasksUtils.GcpCloudTasksClient;
import google.registry.batch.CloudTasksUtils.SerializableCloudTasksClient;
import google.registry.config.CredentialModule.DefaultCredential;
import google.registry.config.CredentialModule.ApplicationDefaultCredential;
import google.registry.config.RegistryConfig.Config;
import google.registry.util.GoogleCredentialsBundle;
import java.io.IOException;
@@ -41,7 +41,7 @@ public abstract class CloudTasksUtilsModule {
// Provides a supplier instead of using a Dagger @Provider because the latter is not serializable.
@Provides
public static Supplier<CloudTasksClient> provideCloudTasksClientSupplier(
@DefaultCredential GoogleCredentialsBundle credentials) {
@ApplicationDefaultCredential GoogleCredentialsBundle credentials) {
return (Supplier<CloudTasksClient> & Serializable)
() -> {
CloudTasksClient client;
@@ -66,38 +66,6 @@ public abstract class CredentialModule {
return GoogleCredentialsBundle.create(credential);
}
/**
* Provides the default {@link GoogleCredentialsBundle} from the Google Cloud runtime.
*
* <p>The credential returned depends on the runtime environment:
*
* <ul>
* <li>On AppEngine, returns the service account credential for
* PROJECT_ID@appspot.gserviceaccount.com
* <li>On Compute Engine, returns the service account credential for
* PROJECT_NUMBER-compute@developer.gserviceaccount.com
* <li>On end user host, this returns the credential downloaded by gcloud. Please refer to <a
* href="https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login">Cloud
* SDK documentation</a> for details.
* </ul>
*/
@DefaultCredential
@Provides
@Singleton
public static GoogleCredentialsBundle provideDefaultCredential(
@Config("defaultCredentialOauthScopes") ImmutableList<String> requiredScopes) {
GoogleCredentials credential;
try {
credential = GoogleCredentials.getApplicationDefault();
} catch (IOException e) {
throw new RuntimeException(e);
}
if (credential.createScopedRequired()) {
credential = credential.createScoped(requiredScopes);
}
return GoogleCredentialsBundle.create(credential);
}
/**
* Provides a {@link GoogleCredentialsBundle} for accessing Google Workspace APIs, such as Drive
* and Sheets.
@@ -162,13 +130,6 @@ public abstract class CredentialModule {
@Retention(RetentionPolicy.RUNTIME)
public @interface ApplicationDefaultCredential {}
/** Dagger qualifier for the Application Default Credential. */
@Qualifier
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Deprecated // Switching to @ApplicationDefaultCredential
public @interface DefaultCredential {}
/** Dagger qualifier for the credential for Google Workspace APIs. */
@Qualifier
@Documented
@@ -108,12 +108,6 @@ public final class RegistryConfig {
return config.gcpProject.projectId;
}
@Provides
@Config("serviceAccountEmails")
public static ImmutableList<String> provideServiceAccountEmails(RegistryConfigSettings config) {
return ImmutableList.copyOf(config.gcpProject.serviceAccountEmails);
}
@Provides
@Config("projectIdNumber")
public static long provideProjectIdNumber(RegistryConfigSettings config) {
@@ -126,6 +120,18 @@ public final class RegistryConfig {
return config.gcpProject.locationId;
}
@Provides
@Config("serviceAccountEmails")
public static ImmutableList<String> provideServiceAccountEmails(RegistryConfigSettings config) {
return ImmutableList.copyOf(config.gcpProject.serviceAccountEmails);
}
@Provides
@Config("defaultServiceAccount")
public static Optional<String> provideDefaultServiceAccount(RegistryConfigSettings config) {
return Optional.ofNullable(config.gcpProject.defaultServiceAccount);
}
/**
* The filename of the logo to be displayed in the header of the registrar console.
*
@@ -55,6 +55,7 @@ public class RegistryConfigSettings {
public String toolsServiceUrl;
public String pubapiServiceUrl;
public List<String> serviceAccountEmails;
public String defaultServiceAccount;
}
/** Configuration options for OAuth settings for authenticating users. */
@@ -27,6 +27,9 @@ gcpProject:
serviceAccountEmails:
- default-service-account-email@email.com
- cloud-scheduler-email@email.com
# The default service account with which the service is running. For example,
# on GAE this would be {project-id}@appspot.gserviceaccount.com
defaultServiceAccount: null
gSuite:
# Publicly accessible domain name of the running G Suite instance.
@@ -140,13 +140,25 @@ public final class TldFanoutAction implements Runnable {
for (String tld : tlds) {
Task task = createTask(tld, flowThruParams);
Task createdTask = cloudTasksUtils.enqueue(queue, task);
outputPayload.append(
String.format(
"- Task: '%s', tld: '%s', endpoint: '%s'\n",
createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri()));
logger.atInfo().log(
"Task: '%s', tld: '%s', endpoint: '%s'.",
createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri());
if (createdTask.hasAppEngineHttpRequest()) {
outputPayload.append(
String.format(
"- Task: '%s', tld: '%s', endpoint: '%s'\n",
createdTask.getName(),
tld,
createdTask.getAppEngineHttpRequest().getRelativeUri()));
logger.atInfo().log(
"Task: '%s', tld: '%s', endpoint: '%s'.",
createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri());
} else {
outputPayload.append(
String.format(
"- Task: '%s', tld: '%s', endpoint: '%s'\n",
createdTask.getName(), tld, createdTask.getHttpRequest().getUrl()));
logger.atInfo().log(
"Task: '%s', tld: '%s', endpoint: '%s'.",
createdTask.getName(), tld, createdTask.getHttpRequest().getUrl());
}
}
response.setContentType(PLAIN_TEXT_UTF_8);
response.setPayload(outputPayload.toString());
@@ -22,7 +22,7 @@ import dagger.Provides;
import dagger.multibindings.IntoMap;
import dagger.multibindings.IntoSet;
import dagger.multibindings.StringKey;
import google.registry.config.CredentialModule.DefaultCredential;
import google.registry.config.CredentialModule.ApplicationDefaultCredential;
import google.registry.config.RegistryConfig.Config;
import google.registry.dns.writer.DnsWriter;
import google.registry.util.GoogleCredentialsBundle;
@@ -35,7 +35,7 @@ public abstract class CloudDnsWriterModule {
@Provides
static Dns provideDns(
@DefaultCredential GoogleCredentialsBundle credentialsBundle,
@ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
@Config("projectId") String projectId,
@Config("cloudDnsRootUrl") Optional<String> rootUrl,
@Config("cloudDnsServicePath") Optional<String> servicePath) {
@@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType;
import google.registry.config.CredentialModule.DefaultCredential;
import google.registry.config.CredentialModule.ApplicationDefaultCredential;
import google.registry.util.GoogleCredentialsBundle;
import java.io.IOException;
import java.io.InputStream;
@@ -64,7 +64,7 @@ public class GcsUtils implements Serializable {
}
@Inject
public GcsUtils(@DefaultCredential GoogleCredentialsBundle credentialsBundle) {
public GcsUtils(@ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle) {
this(
StorageOptions.newBuilder()
.setCredentials(credentialsBundle.getGoogleCredentials())
@@ -1,275 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.common;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryEnvironment;
import google.registry.model.CacheUtils;
import google.registry.model.annotations.DeleteAfterMigration;
import java.time.Duration;
import java.util.Arrays;
import javax.persistence.Entity;
import javax.persistence.PersistenceException;
import org.joda.time.DateTime;
/**
* A wrapper object representing the stage-to-time mapping of the Registry 3.0 Cloud SQL migration.
*
* <p>The entity is stored in SQL throughout the entire migration so as to have a single point of
* access.
*/
@DeleteAfterMigration
@Entity
public class DatabaseMigrationStateSchedule extends CrossTldSingleton {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static boolean useUncachedForTest = false;
public enum PrimaryDatabase {
CLOUD_SQL,
DATASTORE
}
public enum ReplayDirection {
NO_REPLAY,
DATASTORE_TO_SQL,
SQL_TO_DATASTORE
}
/**
* The current phase of the migration plus information about which database to use and whether or
* not the phase is read-only.
*/
public enum MigrationState {
/** Datastore is the only DB being used. */
DATASTORE_ONLY(PrimaryDatabase.DATASTORE, false, ReplayDirection.NO_REPLAY),
/** Datastore is the primary DB, with changes replicated to Cloud SQL. */
DATASTORE_PRIMARY(PrimaryDatabase.DATASTORE, false, ReplayDirection.DATASTORE_TO_SQL),
/** Datastore is the primary DB, with replication, and async actions are disallowed. */
DATASTORE_PRIMARY_NO_ASYNC(PrimaryDatabase.DATASTORE, false, ReplayDirection.DATASTORE_TO_SQL),
/** Datastore is the primary DB, with replication, and all mutating actions are disallowed. */
DATASTORE_PRIMARY_READ_ONLY(PrimaryDatabase.DATASTORE, true, ReplayDirection.DATASTORE_TO_SQL),
/**
* Cloud SQL is the primary DB, with replication back to Datastore, and all mutating actions are
* disallowed.
*/
SQL_PRIMARY_READ_ONLY(PrimaryDatabase.CLOUD_SQL, true, ReplayDirection.SQL_TO_DATASTORE),
/** Cloud SQL is the primary DB, with changes replicated to Datastore. */
SQL_PRIMARY(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.SQL_TO_DATASTORE),
/** Cloud SQL is the only DB being used. */
SQL_ONLY(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.NO_REPLAY),
/** Toggles SQL Sequence based allocateId */
SEQUENCE_BASED_ALLOCATE_ID(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.NO_REPLAY),
/** Use SQL-based Nordn upload flow instead of the pull queue-based one. */
NORDN_SQL(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.NO_REPLAY),
/** Use SQL-based DNS update flow instead of the pull queue-based one. */
DNS_SQL(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.NO_REPLAY);
private final PrimaryDatabase primaryDatabase;
private final boolean isReadOnly;
private final ReplayDirection replayDirection;
public PrimaryDatabase getPrimaryDatabase() {
return primaryDatabase;
}
public boolean isReadOnly() {
return isReadOnly;
}
public ReplayDirection getReplayDirection() {
return replayDirection;
}
MigrationState(
PrimaryDatabase primaryDatabase, boolean isReadOnly, ReplayDirection replayDirection) {
this.primaryDatabase = primaryDatabase;
this.isReadOnly = isReadOnly;
this.replayDirection = replayDirection;
}
}
/**
* Cache of the current migration schedule. The key is meaningless; this is essentially a memoized
* Supplier that can be reset for testing purposes and after writes.
*/
@VisibleForTesting
public static final LoadingCache<
Class<DatabaseMigrationStateSchedule>, TimedTransitionProperty<MigrationState>>
// Each instance should cache the migration schedule for five minutes before reloading
CACHE =
CacheUtils.newCacheBuilder(Duration.ofMinutes(5))
.build(singletonClazz -> DatabaseMigrationStateSchedule.getUncached());
// Restrictions on the state transitions, e.g. no going from DATASTORE_ONLY to SQL_ONLY
private static final ImmutableMultimap<MigrationState, MigrationState> VALID_STATE_TRANSITIONS =
createValidStateTransitions();
/**
* The valid state transitions. Generally, one can advance the state one step or move backward any
* number of steps, as long as the step we're moving back to has the same primary database as the
* one we're in. Otherwise, we must move to the corresponding READ_ONLY stage first.
*/
private static ImmutableMultimap<MigrationState, MigrationState> createValidStateTransitions() {
ImmutableMultimap.Builder<MigrationState, MigrationState> builder =
new ImmutableMultimap.Builder<MigrationState, MigrationState>()
.put(MigrationState.DATASTORE_ONLY, MigrationState.DATASTORE_PRIMARY)
.putAll(
MigrationState.DATASTORE_PRIMARY,
MigrationState.DATASTORE_ONLY,
MigrationState.DATASTORE_PRIMARY_NO_ASYNC)
.putAll(
MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
MigrationState.DATASTORE_ONLY,
MigrationState.DATASTORE_PRIMARY,
MigrationState.DATASTORE_PRIMARY_READ_ONLY)
.putAll(
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
MigrationState.DATASTORE_ONLY,
MigrationState.DATASTORE_PRIMARY,
MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
MigrationState.SQL_PRIMARY_READ_ONLY,
MigrationState.SQL_PRIMARY)
.putAll(
MigrationState.SQL_PRIMARY_READ_ONLY,
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
MigrationState.SQL_PRIMARY)
.putAll(
MigrationState.SQL_PRIMARY,
MigrationState.SQL_PRIMARY_READ_ONLY,
MigrationState.SQL_ONLY)
.putAll(
MigrationState.SQL_ONLY,
MigrationState.SQL_PRIMARY_READ_ONLY,
MigrationState.SQL_PRIMARY)
.putAll(MigrationState.SQL_ONLY, MigrationState.SEQUENCE_BASED_ALLOCATE_ID)
.putAll(MigrationState.SEQUENCE_BASED_ALLOCATE_ID, MigrationState.NORDN_SQL)
.putAll(
MigrationState.NORDN_SQL,
MigrationState.SEQUENCE_BASED_ALLOCATE_ID,
MigrationState.DNS_SQL)
.putAll(MigrationState.DNS_SQL, MigrationState.NORDN_SQL);
// In addition, we can always transition from a state to itself (useful when updating the map).
Arrays.stream(MigrationState.values()).forEach(state -> builder.put(state, state));
return builder.build();
}
// Default map to return if we have never saved any -- only use Datastore.
@VisibleForTesting
public static final TimedTransitionProperty<MigrationState> DEFAULT_TRANSITION_MAP =
TimedTransitionProperty.fromValueMap(
ImmutableSortedMap.of(START_OF_TIME, MigrationState.DATASTORE_ONLY));
@VisibleForTesting
public TimedTransitionProperty<MigrationState> migrationTransitions =
TimedTransitionProperty.withInitialValue(MigrationState.DATASTORE_ONLY);
// Required for Hibernate initialization
protected DatabaseMigrationStateSchedule() {}
@VisibleForTesting
public DatabaseMigrationStateSchedule(
TimedTransitionProperty<MigrationState> migrationTransitions) {
this.migrationTransitions = migrationTransitions;
}
/** Sets and persists to SQL the provided migration transition schedule. */
public static void set(ImmutableSortedMap<DateTime, MigrationState> migrationTransitionMap) {
tm().assertInTransaction();
TimedTransitionProperty<MigrationState> transitions =
TimedTransitionProperty.make(
migrationTransitionMap,
VALID_STATE_TRANSITIONS,
"validStateTransitions",
MigrationState.DATASTORE_ONLY,
"migrationTransitionMap must start with DATASTORE_ONLY");
validateTransitionAtCurrentTime(transitions);
tm().put(new DatabaseMigrationStateSchedule(transitions));
CACHE.invalidateAll();
}
@VisibleForTesting
public static void useUncachedForTest() {
useUncachedForTest = true;
}
/** Loads the currently-set migration schedule from the cache, or the default if none exists. */
public static TimedTransitionProperty<MigrationState> get() {
return CACHE.get(DatabaseMigrationStateSchedule.class);
}
/** Returns the database migration status at the given time. */
public static MigrationState getValueAtTime(DateTime dateTime) {
return useUncachedForTest
? getUncached().getValueAtTime(dateTime)
: get().getValueAtTime(dateTime);
}
/** Loads the currently-set migration schedule from SQL, or the default if none exists. */
@VisibleForTesting
static TimedTransitionProperty<MigrationState> getUncached() {
return tm().transact(
() -> {
try {
return tm().loadSingleton(DatabaseMigrationStateSchedule.class)
.map(s -> s.migrationTransitions)
.orElse(DEFAULT_TRANSITION_MAP);
} catch (PersistenceException e) {
if (!RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)) {
throw e;
}
logger.atWarning().withCause(e).log(
"Error when retrieving migration schedule; this should only happen in tests.");
return DEFAULT_TRANSITION_MAP;
}
});
}
/**
* A provided map of transitions may be valid by itself (i.e. it shifts states properly, doesn't
* skip states, and doesn't backtrack incorrectly) while still being invalid. In addition to the
* transitions in the map being valid, the single transition from the current map at the current
* time to the new map at the current time must also be valid.
*/
private static void validateTransitionAtCurrentTime(
TimedTransitionProperty<MigrationState> newTransitions) {
MigrationState currentValue = getUncached().getValueAtTime(tm().getTransactionTime());
MigrationState nextCurrentValue = newTransitions.getValueAtTime(tm().getTransactionTime());
checkArgument(
VALID_STATE_TRANSITIONS.get(currentValue).contains(nextCurrentValue),
"Cannot transition from current state-as-of-now %s to new state-as-of-now %s",
currentValue,
nextCurrentValue);
}
}
@@ -1,37 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.persistence.converter;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.common.DatabaseMigrationStateSchedule;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import javax.persistence.Converter;
/** JPA converter for {@link DatabaseMigrationStateSchedule} transitions. */
@DeleteAfterMigration
@Converter(autoApply = true)
public class DatabaseMigrationScheduleTransitionConverter
extends TimedTransitionPropertyConverterBase<MigrationState> {
@Override
protected String convertValueToString(MigrationState value) {
return value.name();
}
@Override
protected MigrationState convertStringToValue(String string) {
return MigrationState.valueOf(string);
}
}
@@ -21,7 +21,7 @@ import static google.registry.request.RequestParameters.extractRequiredParameter
import com.google.api.services.dataflow.Dataflow;
import dagger.Module;
import dagger.Provides;
import google.registry.config.CredentialModule.DefaultCredential;
import google.registry.config.CredentialModule.ApplicationDefaultCredential;
import google.registry.config.RegistryConfig.Config;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.Parameter;
@@ -134,7 +134,7 @@ public class ReportingModule {
/** Constructs a {@link Dataflow} API client with default settings. */
@Provides
static Dataflow provideDataflow(
@DefaultCredential GoogleCredentialsBundle credentialsBundle,
@ApplicationDefaultCredential GoogleCredentialsBundle credentialsBundle,
@Config("projectId") String projectId) {
return new Dataflow.Builder(
credentialsBundle.getHttpTransport(),
@@ -35,7 +35,6 @@ import dagger.Lazy;
import dagger.Module;
import dagger.Provides;
import google.registry.config.CredentialModule.ApplicationDefaultCredential;
import google.registry.config.CredentialModule.DefaultCredential;
import google.registry.config.CredentialModule.LocalCredential;
import google.registry.config.CredentialModule.LocalCredentialJson;
import google.registry.config.RegistryConfig.Config;
@@ -222,10 +221,6 @@ public class AuthModule {
@Module
abstract static class LocalCredentialModule {
@Binds
@DefaultCredential
abstract GoogleCredentialsBundle provideLocalCredentialAsDefaultCredential(
@LocalCredential GoogleCredentialsBundle credential);
@Binds
@ApplicationDefaultCredential
@@ -1,34 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import com.beust.jcommander.Parameters;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.common.DatabaseMigrationStateSchedule;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import google.registry.model.common.TimedTransitionProperty;
/** A command to check the current Registry 3.0 migration state of the database. */
@DeleteAfterMigration
@Parameters(separators = " =", commandDescription = "Check current Registry 3.0 migration state")
public class GetDatabaseMigrationStateCommand implements Command {
@Override
public void run() throws Exception {
TimedTransitionProperty<MigrationState> migrationSchedule =
DatabaseMigrationStateSchedule.get();
System.out.printf("Current migration schedule: %s%n", migrationSchedule.toValueMap());
}
}
@@ -15,17 +15,23 @@
package google.registry.tools;
import static google.registry.model.EppResourceUtils.loadByForeignKey;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import google.registry.model.domain.Domain;
import google.registry.persistence.transaction.QueryComposer.Comparator;
import google.registry.util.DomainNameUtils;
import java.util.List;
import java.util.Optional;
/** Command to show a domain resource. */
@Parameters(separators = " =", commandDescription = "Show domain resource(s)")
final class GetDomainCommand extends GetEppResourceCommand {
@Parameter(names = "--show_deleted", description = "Include deleted domains in the print out")
private boolean showDeleted = false;
@Parameter(
description = "Fully qualified domain name(s)",
required = true)
@@ -35,10 +41,24 @@ final class GetDomainCommand extends GetEppResourceCommand {
public void runAndPrint() {
for (String domainName : mainParameters) {
String canonicalDomain = DomainNameUtils.canonicalizeHostname(domainName);
printResource(
"Domain",
canonicalDomain,
loadByForeignKey(Domain.class, canonicalDomain, readTimestamp));
if (showDeleted) {
tm().transact(
() ->
tm()
.createQueryComposer(Domain.class)
.where("domainName", Comparator.EQ, canonicalDomain)
.orderBy("creationTime")
.stream()
.forEach(
d -> {
printResource("Domain", canonicalDomain, Optional.of(d));
}));
} else {
printResource(
"Domain",
canonicalDomain,
loadByForeignKey(Domain.class, canonicalDomain, readTimestamp));
}
}
}
}
@@ -68,7 +68,6 @@ public final class RegistryTool {
.put("get_allocation_token", GetAllocationTokenCommand.class)
.put("get_claims_list", GetClaimsListCommand.class)
.put("get_contact", GetContactCommand.class)
.put("get_database_migration_state", GetDatabaseMigrationStateCommand.class)
.put("get_domain", GetDomainCommand.class)
.put("get_history_entries", GetHistoryEntriesCommand.class)
.put("get_host", GetHostCommand.class)
@@ -98,7 +97,6 @@ public final class RegistryTool {
.put("renew_domain", RenewDomainCommand.class)
.put("save_sql_credential", SaveSqlCredentialCommand.class)
.put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class)
.put("set_database_migration_state", SetDatabaseMigrationStateCommand.class)
.put("setup_ote", SetupOteCommand.class)
.put("uniform_rapid_suspension", UniformRapidSuspensionCommand.class)
.put("unlock_domain", UnlockDomainCommand.class)
@@ -85,7 +85,7 @@ public class ServiceConnection {
private String internalSend(
String endpoint, Map<String, ?> params, MediaType contentType, @Nullable byte[] payload)
throws IOException {
GenericUrl url = new GenericUrl(String.format("%s%s", getServer(), endpoint));
GenericUrl url = new GenericUrl(String.format("%s%s", getServer(service), endpoint));
url.putAll(params);
HttpRequest request =
(payload != null)
@@ -141,7 +141,7 @@ public class ServiceConnection {
return (Map<String, Object>) JSONValue.parse(response.substring(JSON_SAFETY_PREFIX.length()));
}
public URL getServer() {
public static URL getServer(Service service) {
switch (service) {
case DEFAULT:
return RegistryConfig.getDefaultServer();
@@ -1,70 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.collect.ImmutableSortedMap;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.common.DatabaseMigrationStateSchedule;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import google.registry.tools.params.TransitionListParameter.MigrationStateTransitions;
import org.joda.time.DateTime;
/** Command to set the Registry 3.0 database migration state schedule. */
@DeleteAfterMigration
@Parameters(
separators = " =",
commandDescription = "Set the current database migration state schedule.")
public class SetDatabaseMigrationStateCommand extends ConfirmingCommand {
private static final String WARNING_MESSAGE =
"Attempting to change the schedule with an effect that would take place within the next 10 "
+ "minutes. The cache expiration duration is 5 minutes so this MAY BE DANGEROUS.\n";
@Parameter(
names = "--migration_schedule",
converter = MigrationStateTransitions.class,
validateWith = MigrationStateTransitions.class,
required = true,
description =
"Comma-delimited list of database transitions, of the form"
+ " <time>=<migration-state>[,<time>=<migration-state>]*")
ImmutableSortedMap<DateTime, MigrationState> transitionSchedule;
@Override
protected String prompt() {
return tm().transact(
() -> {
StringBuilder result = new StringBuilder();
DateTime now = tm().getTransactionTime();
DateTime nextTransition = transitionSchedule.ceilingKey(now);
if (nextTransition != null && nextTransition.isBefore(now.plusMinutes(10))) {
result.append(WARNING_MESSAGE);
}
return result
.append(String.format("Set new migration state schedule %s?", transitionSchedule))
.toString();
});
}
@Override
protected String execute() {
tm().transact(() -> DatabaseMigrationStateSchedule.set(transitionSchedule));
return String.format("Successfully set new migration state schedule %s", transitionSchedule);
}
}
@@ -190,7 +190,7 @@ final class UpdateDomainCommand extends CreateOrUpdateDomainCommand {
checkArgument(
!domain.getStatusValues().contains(SERVER_UPDATE_PROHIBITED),
"The domain '%s' has status SERVER_UPDATE_PROHIBITED. Verify that you are allowed "
+ "to make updates, and if so, use the domain_unlock command to enable updates.",
+ "to make updates, and if so, use the unlock_domain command to enable updates.",
domainName);
checkArgument(
!domain.getStatusValues().contains(PENDING_DELETE) || forceInPendingDelete,
@@ -19,7 +19,6 @@ import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Ordering;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import google.registry.model.domain.token.AllocationToken.TokenStatus;
import google.registry.model.tld.Tld.TldState;
import org.joda.money.Money;
@@ -73,12 +72,4 @@ public abstract class TransitionListParameter<V> extends KeyValueMapParameter<Da
return TokenStatus.valueOf(value);
}
}
/** Converter-validator for states of the Registry 3.0 database migration. */
public static class MigrationStateTransitions extends TransitionListParameter<MigrationState> {
@Override
protected MigrationState parseValue(String value) {
return MigrationState.valueOf(value);
}
}
}
@@ -42,7 +42,6 @@
<class>google.registry.model.billing.BillingEvent</class>
<class>google.registry.model.billing.BillingRecurrence</class>
<class>google.registry.model.common.Cursor</class>
<class>google.registry.model.common.DatabaseMigrationStateSchedule</class>
<class>google.registry.model.common.DnsRefreshRequest</class>
<class>google.registry.model.console.User</class>
<class>google.registry.model.contact.ContactHistory</class>
@@ -88,7 +87,6 @@
<class>google.registry.persistence.converter.CommandNameSetConverter</class>
<class>google.registry.persistence.converter.CurrencyToBillingConverter</class>
<class>google.registry.persistence.converter.CurrencyUnitConverter</class>
<class>google.registry.persistence.converter.DatabaseMigrationScheduleTransitionConverter</class>
<class>google.registry.persistence.converter.DateTimeConverter</class>
<class>google.registry.persistence.converter.DurationConverter</class>
<class>google.registry.persistence.converter.IdnTableEnumSetConverter</class>
@@ -23,6 +23,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.cloud.tasks.v2.HttpMethod;
import com.google.cloud.tasks.v2.OidcToken;
import com.google.cloud.tasks.v2.Task;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
@@ -46,9 +47,15 @@ public class CloudTasksUtilsTest {
private final LinkedListMultimap<String, String> params = LinkedListMultimap.create();
private final SerializableCloudTasksClient mockClient = mock(SerializableCloudTasksClient.class);
private final FakeClock clock = new FakeClock(DateTime.parse("2021-11-08"));
private final CloudTasksUtils cloudTasksUtils =
private CloudTasksUtils cloudTasksUtils =
new CloudTasksUtils(
new Retrier(new FakeSleeper(clock), 1), clock, "project", "location", mockClient);
new Retrier(new FakeSleeper(clock), 1),
clock,
"project",
"location",
Optional.empty(),
Optional.empty(),
mockClient);
@BeforeEach
void beforeEach() {
@@ -348,4 +355,255 @@ public class CloudTasksUtilsTest {
verify(mockClient).enqueue("project", "location", "test-queue", task1);
verify(mockClient).enqueue("project", "location", "test-queue", task2);
}
@Test
void testSuccess_nonAppEngine_createGetTasks() {
createOidcTasksUtils();
Task task = cloudTasksUtils.createGetTask("/the/path", Service.BACKEND, params);
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
assertThat(task.getHttpRequest().getUrl())
.isEqualTo("https://localhost/the/path?key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createPostTasks() {
createOidcTasksUtils();
Task task = cloudTasksUtils.createPostTask("/the/path", Service.BACKEND, params);
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
assertThat(task.getHttpRequest().getHeadersMap().get("Content-Type"))
.isEqualTo("application/x-www-form-urlencoded");
assertThat(task.getHttpRequest().getBody().toString(StandardCharsets.UTF_8))
.isEqualTo("key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createGetTasks_withNullParams() {
createOidcTasksUtils();
Task task = cloudTasksUtils.createGetTask("/the/path", Service.BACKEND, null);
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createPostTasks_withNullParams() {
createOidcTasksUtils();
Task task = cloudTasksUtils.createPostTask("/the/path", Service.BACKEND, null);
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
assertThat(task.getHttpRequest().getBody().toString(StandardCharsets.UTF_8)).isEmpty();
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createGetTasks_withEmptyParams() {
createOidcTasksUtils();
Task task = cloudTasksUtils.createGetTask("/the/path", Service.BACKEND, ImmutableMultimap.of());
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createPostTasks_withEmptyParams() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createPostTask("/the/path", Service.BACKEND, ImmutableMultimap.of());
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
assertThat(task.getHttpRequest().getBody().toString(StandardCharsets.UTF_8)).isEmpty();
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@SuppressWarnings("ProtoTimestampGetSecondsGetNano")
@Test
void testSuccess_nonAppEngine_createGetTasks_withJitterSeconds() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createGetTaskWithJitter(
"/the/path", Service.BACKEND, params, Optional.of(100));
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
assertThat(task.getHttpRequest().getUrl())
.isEqualTo("https://localhost/the/path?key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
Instant scheduleTime = Instant.ofEpochSecond(task.getScheduleTime().getSeconds());
Instant lowerBoundTime = Instant.ofEpochMilli(clock.nowUtc().getMillis());
Instant upperBound = Instant.ofEpochMilli(clock.nowUtc().plusSeconds(100).getMillis());
assertThat(scheduleTime.isBefore(lowerBoundTime)).isFalse();
assertThat(upperBound.isBefore(scheduleTime)).isFalse();
}
@SuppressWarnings("ProtoTimestampGetSecondsGetNano")
@Test
void testSuccess_nonAppEngine_createPostTasks_withJitterSeconds() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createPostTaskWithJitter(
"/the/path", Service.BACKEND, params, Optional.of(1));
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
assertThat(task.getHttpRequest().getHeadersMap().get("Content-Type"))
.isEqualTo("application/x-www-form-urlencoded");
assertThat(task.getHttpRequest().getBody().toString(StandardCharsets.UTF_8))
.isEqualTo("key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isNotEqualTo(0);
Instant scheduleTime = Instant.ofEpochSecond(task.getScheduleTime().getSeconds());
Instant lowerBoundTime = Instant.ofEpochMilli(clock.nowUtc().getMillis());
Instant upperBound = Instant.ofEpochMilli(clock.nowUtc().plusSeconds(1).getMillis());
assertThat(scheduleTime.isBefore(lowerBoundTime)).isFalse();
assertThat(upperBound.isBefore(scheduleTime)).isFalse();
}
@Test
void testSuccess_nonAppEngine_createPostTasks_withEmptyJitterSeconds() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createPostTaskWithJitter(
"/the/path", Service.BACKEND, params, Optional.empty());
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
assertThat(task.getHttpRequest().getHeadersMap().get("Content-Type"))
.isEqualTo("application/x-www-form-urlencoded");
assertThat(task.getHttpRequest().getBody().toString(StandardCharsets.UTF_8))
.isEqualTo("key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createGetTasks_withEmptyJitterSeconds() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createGetTaskWithJitter(
"/the/path", Service.BACKEND, params, Optional.empty());
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
assertThat(task.getHttpRequest().getUrl())
.isEqualTo("https://localhost/the/path?key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createPostTasks_withZeroJitterSeconds() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createPostTaskWithJitter(
"/the/path", Service.BACKEND, params, Optional.of(0));
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
assertThat(task.getHttpRequest().getHeadersMap().get("Content-Type"))
.isEqualTo("application/x-www-form-urlencoded");
assertThat(task.getHttpRequest().getBody().toString(StandardCharsets.UTF_8))
.isEqualTo("key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createGetTasks_withZeroJitterSeconds() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createGetTaskWithJitter(
"/the/path", Service.BACKEND, params, Optional.of(0));
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
assertThat(task.getHttpRequest().getUrl())
.isEqualTo("https://localhost/the/path?key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createGetTasks_withDelay() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createGetTaskWithDelay(
"/the/path", Service.BACKEND, params, Duration.standardMinutes(10));
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
assertThat(task.getHttpRequest().getUrl())
.isEqualTo("https://localhost/the/path?key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(Instant.ofEpochSecond(task.getScheduleTime().getSeconds()))
.isEqualTo(Instant.ofEpochMilli(clock.nowUtc().plusMinutes(10).getMillis()));
}
@Test
void testSuccess_nonAppEngine_createPostTasks_withDelay() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createPostTaskWithDelay(
"/the/path", Service.BACKEND, params, Duration.standardMinutes(10));
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
assertThat(task.getHttpRequest().getHeadersMap().get("Content-Type"))
.isEqualTo("application/x-www-form-urlencoded");
assertThat(task.getHttpRequest().getBody().toString(StandardCharsets.UTF_8))
.isEqualTo("key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isNotEqualTo(0);
assertThat(Instant.ofEpochSecond(task.getScheduleTime().getSeconds()))
.isEqualTo(Instant.ofEpochMilli(clock.nowUtc().plusMinutes(10).getMillis()));
}
@Test
void testSuccess_nonAppEngine_createPostTasks_withZeroDelay() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createPostTaskWithDelay(
"/the/path", Service.BACKEND, params, Duration.ZERO);
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
assertThat(task.getHttpRequest().getUrl()).isEqualTo("https://localhost/the/path");
assertThat(task.getHttpRequest().getHeadersMap().get("Content-Type"))
.isEqualTo("application/x-www-form-urlencoded");
assertThat(task.getHttpRequest().getBody().toString(StandardCharsets.UTF_8))
.isEqualTo("key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
@Test
void testSuccess_nonAppEngine_createGetTasks_withZeroDelay() {
createOidcTasksUtils();
Task task =
cloudTasksUtils.createGetTaskWithDelay("/the/path", Service.BACKEND, params, Duration.ZERO);
assertThat(task.getHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
assertThat(task.getHttpRequest().getUrl())
.isEqualTo("https://localhost/the/path?key1=val1&key2=val2&key1=val3");
verifyOidcToken(task);
assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
}
private void createOidcTasksUtils() {
cloudTasksUtils =
new CloudTasksUtils(
new Retrier(new FakeSleeper(clock), 1),
clock,
"project",
"location",
Optional.of("defaultServiceAccount"),
Optional.of("iapClientId"),
mockClient);
}
private void verifyOidcToken(Task task) {
assertThat(task.getHttpRequest().getOidcToken())
.isEqualTo(
OidcToken.newBuilder()
.setServiceAccountEmail("defaultServiceAccount")
.setAudience("iapClientId")
.build());
}
}
@@ -83,7 +83,7 @@ class BillingEventTest {
assertThat(invoiceKey.startDate()).isEqualTo("2017-10-01");
assertThat(invoiceKey.endDate()).isEqualTo("2022-09-30");
assertThat(invoiceKey.productAccountKey()).isEqualTo("12345-CRRHELLO");
assertThat(invoiceKey.usageGroupingKey()).isEqualTo("myRegistrar - test");
assertThat(invoiceKey.usageGroupingKey()).isEqualTo("myRegistrar");
assertThat(invoiceKey.description()).isEqualTo("RENEW | TLD: test | TERM: 5-year");
assertThat(invoiceKey.unitPrice()).isEqualTo(20.5);
assertThat(invoiceKey.unitPriceCurrency()).isEqualTo("USD");
@@ -104,7 +104,7 @@ class BillingEventTest {
assertThat(invoiceKey.toCsv(3L))
.isEqualTo(
"2017-10-01,2022-09-30,12345-CRRHELLO,61.50,USD,10125,1,PURCHASE,"
+ "myRegistrar - test,3,RENEW | TLD: test | TERM: 5-year,20.50,USD,");
+ "myRegistrar,3,RENEW | TLD: test | TERM: 5-year,20.50,USD,");
}
@Test
@@ -114,7 +114,7 @@ class BillingEventTest {
assertThat(invoiceKey.toCsv(3L))
.isEqualTo(
"2017-10-01,,12345-CRRHELLO,61.50,USD,10125,1,PURCHASE,"
+ "myRegistrar - test,3,RENEW | TLD: test | TERM: 0-year,20.50,USD,");
+ "myRegistrar,3,RENEW | TLD: test | TERM: 0-year,20.50,USD,");
}
@Test
@@ -224,13 +224,13 @@ class InvoicingPipelineTest {
private static final ImmutableList<String> EXPECTED_INVOICE_OUTPUT =
ImmutableList.of(
"2017-10-01,2020-09-30,234,41.00,USD,10125,1,PURCHASE,theRegistrar - test,2,"
"2017-10-01,2020-09-30,234,41.00,USD,10125,1,PURCHASE,theRegistrar,2,"
+ "RENEW | TLD: test | TERM: 3-year,20.50,USD,",
"2017-10-01,2022-09-30,234,70.00,JPY,10125,1,PURCHASE,theRegistrar - hello,1,"
"2017-10-01,2022-09-30,234,70.00,JPY,10125,1,PURCHASE,theRegistrar,1,"
+ "CREATE | TLD: hello | TERM: 5-year,70.00,JPY,",
"2017-10-01,,234,20.00,USD,10125,1,PURCHASE,theRegistrar - test,1,"
"2017-10-01,,234,20.00,USD,10125,1,PURCHASE,theRegistrar,1,"
+ "SERVER_STATUS | TLD: test | TERM: 0-year,20.00,USD,",
"2017-10-01,2018-09-30,456,20.50,USD,10125,1,PURCHASE,bestdomains - test,1,"
"2017-10-01,2018-09-30,456,20.50,USD,10125,1,PURCHASE,bestdomains,1,"
+ "RENEW | TLD: test | TERM: 1-year,20.50,USD,116688");
private final InvoicingPipelineOptions options =
@@ -1,187 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.common;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_ONLY;
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY;
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC;
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_READ_ONLY;
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_ONLY;
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY;
import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY_READ_ONLY;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static org.junit.Assert.assertThrows;
import com.google.common.collect.ImmutableSortedMap;
import google.registry.model.EntityTestCase;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import google.registry.testing.DatabaseHelper;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/** Tests for {@link DatabaseMigrationStateSchedule}. */
public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
@BeforeEach
void beforeEach() {
fakeClock.setAutoIncrementByOneMilli();
}
@AfterEach
void afterEach() {
DatabaseHelper.removeDatabaseMigrationSchedule();
}
@Test
void testEmpty_returnsDatastoreOnlyMap() {
assertThat(DatabaseMigrationStateSchedule.getUncached())
.isEqualTo(DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP);
}
@Test
void testValidTransitions() {
// First, verify that no-ops are safe
for (MigrationState migrationState : MigrationState.values()) {
runValidTransition(migrationState, migrationState);
}
// Next, the transitions that will actually cause a change
runValidTransition(DATASTORE_ONLY, DATASTORE_PRIMARY);
runValidTransition(DATASTORE_PRIMARY, DATASTORE_ONLY);
runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC);
runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY);
runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_ONLY);
runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY);
runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY_NO_ASYNC);
runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY);
runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY);
runValidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_PRIMARY_READ_ONLY);
runValidTransition(SQL_PRIMARY_READ_ONLY, SQL_PRIMARY);
runValidTransition(SQL_PRIMARY, SQL_PRIMARY_READ_ONLY);
runValidTransition(SQL_PRIMARY, SQL_ONLY);
runValidTransition(SQL_ONLY, SQL_PRIMARY);
}
@Test
void testInvalidTransitions() {
runInvalidTransition(DATASTORE_ONLY, DATASTORE_PRIMARY_READ_ONLY);
runInvalidTransition(DATASTORE_ONLY, SQL_PRIMARY_READ_ONLY);
runInvalidTransition(DATASTORE_ONLY, SQL_PRIMARY);
runInvalidTransition(DATASTORE_ONLY, SQL_ONLY);
runInvalidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY);
runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY_READ_ONLY);
runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY);
runInvalidTransition(DATASTORE_PRIMARY, SQL_ONLY);
runInvalidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_ONLY);
runInvalidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_ONLY);
runInvalidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_PRIMARY);
runInvalidTransition(SQL_PRIMARY_READ_ONLY, SQL_ONLY);
runInvalidTransition(SQL_PRIMARY, DATASTORE_ONLY);
runInvalidTransition(SQL_PRIMARY, DATASTORE_PRIMARY);
runInvalidTransition(SQL_PRIMARY, DATASTORE_PRIMARY_READ_ONLY);
runInvalidTransition(SQL_ONLY, DATASTORE_ONLY);
runInvalidTransition(SQL_ONLY, DATASTORE_PRIMARY);
runInvalidTransition(SQL_ONLY, DATASTORE_PRIMARY_READ_ONLY);
}
@Test
void testFailure_newMapImpliesInvalidChangeNow() {
DateTime startTime = fakeClock.nowUtc();
fakeClock.advanceBy(Duration.standardHours(6));
// The new map is valid by itself, but not with the current state of DATASTORE_ONLY because the
// new map implies that the current state is DATASTORE_PRIMARY_READ_ONLY
ImmutableSortedMap<DateTime, MigrationState> nowInvalidMap =
ImmutableSortedMap.<DateTime, MigrationState>naturalOrder()
.put(START_OF_TIME, DATASTORE_ONLY)
.put(startTime.plusHours(1), DATASTORE_PRIMARY)
.put(startTime.plusHours(2), DATASTORE_PRIMARY_NO_ASYNC)
.put(startTime.plusHours(3), DATASTORE_PRIMARY_READ_ONLY)
.build();
IllegalArgumentException thrown =
assertThrows(
IllegalArgumentException.class,
() -> tm().transact(() -> DatabaseMigrationStateSchedule.set(nowInvalidMap)));
assertThat(thrown)
.hasMessageThat()
.isEqualTo(
"Cannot transition from current state-as-of-now DATASTORE_ONLY "
+ "to new state-as-of-now DATASTORE_PRIMARY_READ_ONLY");
}
@Test
void testFailure_notInTransaction() {
IllegalStateException thrown =
assertThrows(
IllegalStateException.class,
() ->
DatabaseMigrationStateSchedule.set(
DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP.toValueMap()));
assertThat(thrown).hasMessageThat().isEqualTo("Not in a transaction");
}
private void runValidTransition(MigrationState from, MigrationState to) {
ImmutableSortedMap<DateTime, MigrationState> transitions =
createMapEndingWithTransition(from, to);
tm().transact(() -> DatabaseMigrationStateSchedule.set(transitions));
assertThat(DatabaseMigrationStateSchedule.getUncached().toValueMap())
.containsExactlyEntriesIn(transitions);
}
private void runInvalidTransition(MigrationState from, MigrationState to) {
ImmutableSortedMap<DateTime, MigrationState> transitions =
createMapEndingWithTransition(from, to);
IllegalArgumentException thrown =
assertThrows(
IllegalArgumentException.class,
() -> tm().transact(() -> DatabaseMigrationStateSchedule.set(transitions)));
assertThat(thrown)
.hasMessageThat()
.isEqualTo(
String.format("validStateTransitions map cannot transition from %s to %s.", from, to));
}
// Create a transition map that is valid up to the "from" transition, then add the "to" transition
private ImmutableSortedMap<DateTime, MigrationState> createMapEndingWithTransition(
MigrationState from, MigrationState to) {
ImmutableSortedMap.Builder<DateTime, MigrationState> builder =
ImmutableSortedMap.naturalOrder();
builder.put(START_OF_TIME, DATASTORE_ONLY);
MigrationState[] allMigrationStates = MigrationState.values();
for (int i = 0; i < allMigrationStates.length; i++) {
builder.put(fakeClock.nowUtc().plusMinutes(i), allMigrationStates[i]);
if (allMigrationStates[i].equals(from)) {
break;
}
}
builder.put(fakeClock.nowUtc().plusDays(1), to);
return builder.build();
}
}
@@ -1,88 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.persistence.converter;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.testing.DatabaseHelper.insertInDb;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import com.google.common.collect.ImmutableSortedMap;
import google.registry.model.ImmutableObject;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import google.registry.model.common.TimedTransitionProperty;
import google.registry.persistence.transaction.JpaTestExtensions;
import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension;
import javax.persistence.Entity;
import javax.persistence.Id;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
/** Unit tests for {@link DatabaseMigrationScheduleTransitionConverter}. */
public class DatabaseMigrationScheduleTransitionConverterTest {
@RegisterExtension
public final JpaUnitTestExtension jpa =
new JpaTestExtensions.Builder()
.withEntityClass(DatabaseMigrationScheduleTransitionConverterTestEntity.class)
.buildUnitTestExtension();
private static final ImmutableSortedMap<DateTime, MigrationState> values =
ImmutableSortedMap.of(
START_OF_TIME,
MigrationState.DATASTORE_ONLY,
DateTime.parse("2001-01-01T00:00:00.0Z"),
MigrationState.DATASTORE_PRIMARY,
DateTime.parse("2002-01-01T01:00:00.0Z"),
MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
DateTime.parse("2002-01-01T02:00:00.0Z"),
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
DateTime.parse("2002-01-02T00:00:00.0Z"),
MigrationState.SQL_PRIMARY,
DateTime.parse("2002-01-03T00:00:00.0Z"),
MigrationState.SQL_ONLY);
@Test
void roundTripConversion_returnsSameTimedTransitionProperty() {
TimedTransitionProperty<MigrationState> timedTransitionProperty =
TimedTransitionProperty.fromValueMap(values);
DatabaseMigrationScheduleTransitionConverterTestEntity testEntity =
new DatabaseMigrationScheduleTransitionConverterTestEntity(timedTransitionProperty);
insertInDb(testEntity);
DatabaseMigrationScheduleTransitionConverterTestEntity persisted =
tm().transact(
() ->
tm().getEntityManager()
.find(DatabaseMigrationScheduleTransitionConverterTestEntity.class, "id"));
assertThat(persisted.timedTransitionProperty).containsExactlyEntriesIn(timedTransitionProperty);
}
@Entity
private static class DatabaseMigrationScheduleTransitionConverterTestEntity
extends ImmutableObject {
@Id String name = "id";
TimedTransitionProperty<MigrationState> timedTransitionProperty;
private DatabaseMigrationScheduleTransitionConverterTestEntity() {}
private DatabaseMigrationScheduleTransitionConverterTestEntity(
TimedTransitionProperty<MigrationState> timedTransitionProperty) {
this.timedTransitionProperty = timedTransitionProperty;
}
}
}
@@ -39,17 +39,8 @@ import org.junit.jupiter.api.extension.ExtensionContext;
*/
public class JpaEntityCoverageExtension implements BeforeEachCallback, AfterEachCallback {
private static final ImmutableSet<String> IGNORE_ENTITIES =
ImmutableSet.of(
// DatabaseMigrationStateSchedule is persisted in tests, however any test that sets it
// needs to remove it in order to avoid affecting any other tests running in the same JVM.
// TODO(gbrodman): remove this when we implement proper read-only modes for the
// transaction managers.
"DatabaseMigrationStateSchedule");
public static final ImmutableSet<Class<?>> ALL_JPA_ENTITIES =
PersistenceXmlUtility.getManagedClasses().stream()
.filter(e -> !IGNORE_ENTITIES.contains(e.getSimpleName()))
.filter(e -> e.isAnnotationPresent(Entity.class))
.filter(e -> !e.isAnnotationPresent(DiscriminatorValue.class))
.collect(ImmutableSet.toImmutableSet());
@@ -58,6 +58,7 @@ import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
@@ -103,6 +104,8 @@ public class CloudTasksHelper implements Serializable {
clock,
PROJECT_ID,
LOCATION_ID,
Optional.empty(),
Optional.empty(),
new FakeCloudTasksClient());
testTasks.put(instanceId, Multimaps.synchronizedListMultimap(LinkedListMultimap.create()));
}
@@ -71,8 +71,6 @@ import google.registry.model.billing.BillingBase.RenewalPriceBehavior;
import google.registry.model.billing.BillingCancellation;
import google.registry.model.billing.BillingEvent;
import google.registry.model.billing.BillingRecurrence;
import google.registry.model.common.DatabaseMigrationStateSchedule;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import google.registry.model.common.DnsRefreshRequest;
import google.registry.model.contact.Contact;
import google.registry.model.contact.ContactAuthInfo;
@@ -123,7 +121,6 @@ import org.joda.money.CurrencyUnit;
import org.joda.money.Money;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Duration;
/** Static utils for setting up test resources. */
public final class DatabaseHelper {
@@ -1316,46 +1313,6 @@ public final class DatabaseHelper {
return entity;
}
/**
* Sets a SQL_PRIMARY state on the {@link DatabaseMigrationStateSchedule}.
*
* <p>In order to allow for tests to manipulate the clock how they need, we start the transitions
* one millisecond after the clock's current time (in case the clock's current value is
* START_OF_TIME). We then advance the clock one second so that we're in the SQL_PRIMARY phase.
*
* <p>We must use the current time, otherwise the setting of the migration state will fail due to
* an invalid transition.
*/
public static void setMigrationScheduleToSqlPrimary(FakeClock fakeClock) {
DateTime now = fakeClock.nowUtc();
tm().transact(
() ->
DatabaseMigrationStateSchedule.set(
ImmutableSortedMap.of(
START_OF_TIME,
MigrationState.DATASTORE_ONLY,
now.plusMillis(1),
MigrationState.DATASTORE_PRIMARY,
now.plusMillis(2),
MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
now.plusMillis(3),
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
now.plusMillis(4),
MigrationState.SQL_PRIMARY)));
fakeClock.advanceBy(Duration.standardSeconds(1));
}
/** Removes the database migration schedule, in essence transitioning to DATASTORE_ONLY. */
public static void removeDatabaseMigrationSchedule() {
// use the raw calls because going SQL_PRIMARY -> DATASTORE_ONLY is not valid
tm().transact(
() ->
tm().put(
new DatabaseMigrationStateSchedule(
DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP)));
DatabaseMigrationStateSchedule.CACHE.invalidateAll();
}
private static ImmutableList<String> getDnsRefreshRequests(TargetType type, String... names) {
return tm().transact(
() ->
@@ -1,66 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static google.registry.model.common.DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import com.google.common.collect.ImmutableSortedMap;
import google.registry.model.common.DatabaseMigrationStateSchedule;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import google.registry.testing.DatabaseHelper;
import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
/** Tests for {@link GetDatabaseMigrationStateCommand}. */
public class GetDatabaseMigrationStateCommandTest
extends CommandTestCase<GetDatabaseMigrationStateCommand> {
@AfterEach
void afterEach() {
DatabaseHelper.removeDatabaseMigrationSchedule();
}
@Test
void testInitial_returnsDatastoreOnly() throws Exception {
runCommand();
assertStdoutIs(
String.format("Current migration schedule: %s\n", DEFAULT_TRANSITION_MAP.toValueMap()));
}
@Test
void testFullSchedule() throws Exception {
DateTime now = fakeClock.nowUtc();
ImmutableSortedMap<DateTime, MigrationState> transitions =
ImmutableSortedMap.of(
START_OF_TIME,
MigrationState.DATASTORE_ONLY,
now.plusHours(1),
MigrationState.DATASTORE_PRIMARY,
now.plusHours(2),
MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
now.plusHours(3),
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
now.plusHours(4),
MigrationState.SQL_PRIMARY,
now.plusHours(5),
MigrationState.SQL_ONLY);
tm().transact(() -> DatabaseMigrationStateSchedule.set(transitions));
runCommand();
assertStdoutIs(String.format("Current migration schedule: %s\n", transitions));
}
}
@@ -116,4 +116,25 @@ class GetDomainCommandTest extends CommandTestCase<GetDomainCommand> {
assertInStdout("domainName=example.tld");
assertInStdout("Domain 'example.com' does not exist or is deleted");
}
@Test
void testSuccess_printDeletedDomain() throws Exception {
persistDeletedDomain("example.tld", fakeClock.nowUtc().minusDays(1));
runCommand("--show_deleted", "example.tld");
assertInStdout("domainName=example.tld");
assertInStdout("Websafe key: kind:Domain@sql:rO0ABXQABTItVExE");
}
@Test
void testSuccess_printsEntireDomainHistory() throws Exception {
persistActiveDomain("example.tld");
persistDeletedDomain("example.tld", fakeClock.nowUtc().minusDays(1));
runCommand("--show_deleted", "example.tld");
assertInStdout("domainName=example.tld");
// Active
assertInStdout("Websafe key: kind:Domain@sql:rO0ABXQABTItVExE");
// Deleted
assertInStdout("Websafe key: kind:Domain@sql:rO0ABXQABTQtVExE");
}
}
@@ -1,181 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.model.common.DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.beust.jcommander.ParameterException;
import com.google.common.collect.ImmutableSortedMap;
import google.registry.model.common.DatabaseMigrationStateSchedule;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import google.registry.testing.DatabaseHelper;
import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
/** Tests for {@link SetDatabaseMigrationStateCommand}. */
public class SetDatabaseMigrationStateCommandTest
extends CommandTestCase<SetDatabaseMigrationStateCommand> {
@AfterEach
void afterEach() {
DatabaseHelper.removeDatabaseMigrationSchedule();
}
@Test
void testSuccess_setsBasicSchedule() throws Exception {
assertThat(DatabaseMigrationStateSchedule.get()).isEqualTo(DEFAULT_TRANSITION_MAP);
assertThat(tm().transact(() -> tm().loadSingleton(DatabaseMigrationStateSchedule.class)))
.isEmpty();
runCommandForced("--migration_schedule=1970-01-01T00:00:00.000Z=DATASTORE_ONLY");
tm().transact(
() ->
assertThat(
tm().loadSingleton(DatabaseMigrationStateSchedule.class)
.get()
.migrationTransitions)
.isEqualTo(DEFAULT_TRANSITION_MAP));
assertThat(DatabaseMigrationStateSchedule.get()).isEqualTo(DEFAULT_TRANSITION_MAP);
}
@Test
void testSuccess_fullSchedule() throws Exception {
DateTime now = fakeClock.nowUtc();
DateTime datastorePrimary = now.plusHours(1);
DateTime datastorePrimaryNoAsync = now.plusHours(2);
DateTime datastorePrimaryReadOnly = now.plusHours(3);
DateTime sqlPrimary = now.plusHours(4);
DateTime sqlOnly = now.plusHours(5);
runCommandForced(
String.format(
"--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
+ "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY,"
+ "%s=SQL_PRIMARY,%s=SQL_ONLY",
START_OF_TIME,
datastorePrimary,
datastorePrimaryNoAsync,
datastorePrimaryReadOnly,
sqlPrimary,
sqlOnly));
assertThat(DatabaseMigrationStateSchedule.get().toValueMap())
.containsExactlyEntriesIn(
ImmutableSortedMap.of(
START_OF_TIME,
MigrationState.DATASTORE_ONLY,
datastorePrimary,
MigrationState.DATASTORE_PRIMARY,
datastorePrimaryNoAsync,
MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
datastorePrimaryReadOnly,
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
sqlPrimary,
MigrationState.SQL_PRIMARY,
sqlOnly,
MigrationState.SQL_ONLY));
}
@Test
void testSuccess_warnsOnChangeSoon() throws Exception {
DateTime now = fakeClock.nowUtc();
runCommandForced(
String.format(
"--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY",
START_OF_TIME, now.plusMinutes(1)));
assertThat(DatabaseMigrationStateSchedule.get().toValueMap())
.containsExactlyEntriesIn(
ImmutableSortedMap.of(
START_OF_TIME,
MigrationState.DATASTORE_ONLY,
now.plusMinutes(1),
MigrationState.DATASTORE_PRIMARY));
assertInStdout("MAY BE DANGEROUS");
}
@Test
void testSuccess_goesBackward() throws Exception {
DateTime now = fakeClock.nowUtc();
runCommandForced(
String.format(
"--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
+ "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY,"
+ "%s=DATASTORE_PRIMARY",
START_OF_TIME, now.plusHours(1), now.plusHours(2), now.plusHours(3), now.plusHours(4)));
assertThat(DatabaseMigrationStateSchedule.get().toValueMap())
.containsExactlyEntriesIn(
ImmutableSortedMap.of(
START_OF_TIME,
MigrationState.DATASTORE_ONLY,
now.plusHours(1),
MigrationState.DATASTORE_PRIMARY,
now.plusHours(2),
MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
now.plusHours(3),
MigrationState.DATASTORE_PRIMARY_READ_ONLY,
now.plusHours(4),
MigrationState.DATASTORE_PRIMARY));
}
@Test
void testFailure_invalidTransition() {
IllegalArgumentException thrown =
assertThrows(
IllegalArgumentException.class,
() ->
runCommandForced(
String.format(
"--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY_READ_ONLY",
START_OF_TIME, START_OF_TIME.plusHours(1))));
assertThat(thrown)
.hasMessageThat()
.isEqualTo(
"validStateTransitions map cannot transition from DATASTORE_ONLY "
+ "to DATASTORE_PRIMARY_READ_ONLY.");
}
@Test
void testFailure_invalidTransitionFromOldToNew() {
// The map we pass in is valid by itself, but we can't go from DATASTORE_ONLY now to
// DATASTORE_PRIMARY_READ_ONLY now
DateTime now = fakeClock.nowUtc();
IllegalArgumentException thrown =
assertThrows(
IllegalArgumentException.class,
() ->
runCommandForced(
String.format(
"--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
+ "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY",
START_OF_TIME, now.minusHours(3), now.minusHours(2), now.minusHours(1))));
assertThat(thrown)
.hasMessageThat()
.isEqualTo(
"Cannot transition from current state-as-of-now DATASTORE_ONLY "
+ "to new state-as-of-now DATASTORE_PRIMARY_READ_ONLY");
}
@Test
void testFailure_invalidParams() {
assertThrows(ParameterException.class, this::runCommandForced);
assertThrows(ParameterException.class, () -> runCommandForced("--migration_schedule=FOOBAR"));
assertThrows(
ParameterException.class,
() -> runCommandForced("--migration_schedule=1970-01-01T00:00:00.000Z=FOOBAR"));
}
}
@@ -241,12 +241,6 @@
primary key (scope, type)
);
create table "DatabaseMigrationStateSchedule" (
id int8 not null,
migration_transitions hstore,
primary key (id)
);
create table "DelegationSignerData" (
algorithm int4 not null,
digest bytea not null,
+3 -4
View File
@@ -4,7 +4,7 @@ There are multiple different kinds of configuration that go into getting a
working registry system up and running. Broadly speaking, configuration works in
two ways -- globally, for the entire sytem, and per-TLD. Global configuration is
managed by editing code and deploying a new version, whereas per-TLD
configuration is data that lives in the database in `Registry` entities, and is
configuration is data that lives in the database in `Tld` entities, and is
updated by running `nomulus` commands without having to deploy a new version.
## Initial configuration
@@ -177,7 +177,7 @@ SecretManager to configure accordingly, for example:
## Per-TLD configuration
`Registry` entities, which are persisted to the database, are used for per-TLD
`Tld` entities, which are persisted to the database, are used for per-TLD
configuration. They contain any kind of configuration that is specific to a TLD,
such as the create/renew price of a domain name, the pricing engine
implementation, the DNS writer implementation, whether escrow exports are
@@ -194,8 +194,7 @@ and thus do not require code pushes to update.
## Cloud SQL Configuration
Nomulus is in the process of being ported to Cloud SQL. As such, parts of the
system already require access to Cloud SQL and the necessary configuration
Nomulus requires access to Cloud SQL and thus the necessary configuration
must be applied.
### Create Postgres Cloud SQL Instance