1
0
mirror of https://github.com/google/nomulus synced 2026-06-09 16:33:02 +00:00

Compare commits

...

4 Commits

Author SHA1 Message Date
gbrodman 7c364b4471 Add SetSqlReplayCheckpoint command for SQL replay (#895)
* Add SetSqlReplayCheckpoint command for SQL replay

We should set this to the same time that we initially populate the SQL
database from Datastore.
2020-12-11 17:41:06 -05:00
Shicong Huang b5137c3d05 Convert HostResourceTest to work with Cloud SQL (#905) 2020-12-11 13:17:55 -05:00
Ben McIlwain 6a9929019a Use 10 workers instead of the default 100 for re-save all EPP resources (#904)
* Use 10 workers instead of the default 100 for re-save all EPP resources

The intended/desired effect is to have a larger number of GCS commit log diffs
spread out over a longer period of time, with each diff itself being
significantly smaller. This should retain roughly the same amount of total work
for the async Cloud SQL replication action to have to deal with, but spread
across 10X as much time.
2020-12-10 15:25:25 -05:00
Weimin Yu 83ed448741 Add a credential store backed by Secret Manager (#901)
* Add a credential store backed by Secret Manager

Added a SqlCredentialStore that stores user credentials with one level
of indirection: for each credential, an addtional secret is used to
identify the 'live' version of the credential. This is a work in
progress and the overall design is explained in
go/dr-sql-security.

Also added two nomulus commands for credential management. They are
stop-gap measures that will be deprecated by the planned privilege
management system.
2020-12-10 11:29:44 -05:00
19 changed files with 828 additions and 67 deletions
@@ -50,11 +50,21 @@ public class ResaveAllEppResourcesAction implements Runnable {
@Inject Response response;
@Inject ResaveAllEppResourcesAction() {}
/**
* The number of shards to run the map-only mapreduce on.
*
* <p>This is less than the default of 100 because we only run this action monthly and can afford
* it being slower, but we don't want to write out lots of large commit logs in a short period of
* time because they make the Cloud SQL migration tougher.
*/
private static final int NUM_SHARDS = 10;
@Override
public void run() {
mrRunner
.setJobName("Re-save all EPP resources")
.setModuleName("backend")
.setDefaultMapShards(NUM_SHARDS)
.runMapOnly(
new ResaveAllEppResourcesActionMapper(),
ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))
@@ -415,6 +415,14 @@ public final class RegistryConfig {
return config.cloudSql.instanceConnectionName;
}
@Provides
@Config("cloudSqlDbInstanceName")
public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
// Format of instanceConnectionName: project-id:region:instance-name
int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);
}
@Provides
@Config("cloudDnsRootUrl")
public static Optional<String> getCloudDnsRootUrl(RegistryConfigSettings config) {
@@ -22,6 +22,9 @@ import java.util.Optional;
/** A Cloud Secret Manager client for Nomulus, bound to a specific GCP project. */
public interface SecretManagerClient {
/** Returns the project name with which this client is associated. */
String getProject();
/**
* Creates a new secret in the Cloud Secret Manager with no data.
*
@@ -32,6 +35,9 @@ public interface SecretManagerClient {
*/
void createSecret(String secretId);
/** Checks if a secret with the given {@code secretId} already exists. */
boolean secretExists(String secretId);
/** Returns all secret IDs in the Cloud Secret Manager. */
Iterable<String> listSecrets();
@@ -67,6 +73,24 @@ public interface SecretManagerClient {
*/
String getSecretData(String secretId, Optional<String> version);
/**
* Enables a secret version.
*
* @param secretId The ID of the secret
* @param version The version of the secret to fetch. If not provided, the {@code latest} version
* will be returned
*/
void enableSecretVersion(String secretId, String version);
/**
* Disables a secret version.
*
* @param secretId The ID of the secret
* @param version The version of the secret to fetch. If not provided, the {@code latest} version
* will be returned
*/
void disableSecretVersion(String secretId, String version);
/**
* Destroys a secret version.
*
@@ -29,12 +29,11 @@ import com.google.cloud.secretmanager.v1.SecretName;
import com.google.cloud.secretmanager.v1.SecretPayload;
import com.google.cloud.secretmanager.v1.SecretVersion;
import com.google.cloud.secretmanager.v1.SecretVersionName;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams;
import com.google.protobuf.ByteString;
import google.registry.util.Retrier;
import java.util.Optional;
import java.util.concurrent.Callable;
import javax.inject.Inject;
/** Implements {@link SecretManagerClient} on Google Cloud Platform. */
public class SecretManagerClientImpl implements SecretManagerClient {
@@ -42,13 +41,17 @@ public class SecretManagerClientImpl implements SecretManagerClient {
private final SecretManagerServiceClient csmClient;
private final Retrier retrier;
@Inject
SecretManagerClientImpl(String project, SecretManagerServiceClient csmClient, Retrier retrier) {
this.project = project;
this.csmClient = csmClient;
this.retrier = retrier;
}
@Override
public String getProject() {
return project;
}
@Override
public void createSecret(String secretId) {
checkNotNull(secretId, "secretId");
@@ -57,12 +60,25 @@ public class SecretManagerClientImpl implements SecretManagerClient {
() -> csmClient.createSecret(ProjectName.of(project), secretId, secretSettings));
}
@Override
public boolean secretExists(String secretId) {
checkNotNull(secretId, "secretId");
try {
callSecretManager(() -> csmClient.getSecret(SecretName.of(project, secretId)));
return true;
} catch (NoSuchSecretResourceException e) {
return false;
}
}
@Override
public Iterable<String> listSecrets() {
ListSecretsPagedResponse response =
callSecretManager(() -> csmClient.listSecrets(ProjectName.of(project)));
return Iterables.transform(
response.iterateAll(), secret -> SecretName.parse(secret.getName()).getSecret());
return () ->
Streams.stream(response.iterateAll())
.map(secret -> SecretName.parse(secret.getName()).getSecret())
.iterator();
}
@Override
@@ -70,8 +86,10 @@ public class SecretManagerClientImpl implements SecretManagerClient {
checkNotNull(secretId, "secretId");
ListSecretVersionsPagedResponse response =
callSecretManager(() -> csmClient.listSecretVersions(SecretName.of(project, secretId)));
return Iterables.transform(
response.iterateAll(), SecretManagerClientImpl::toSecretVersionState);
return () ->
Streams.stream(response.iterateAll())
.map(SecretManagerClientImpl::toSecretVersionState)
.iterator();
}
private static SecretVersionState toSecretVersionState(SecretVersion secretVersion) {
@@ -108,6 +126,22 @@ public class SecretManagerClientImpl implements SecretManagerClient {
.toStringUtf8());
}
@Override
public void enableSecretVersion(String secretId, String version) {
checkNotNull(secretId, "secretId");
checkNotNull(version, "version");
callSecretManager(
() -> csmClient.enableSecretVersion(SecretVersionName.of(project, secretId, version)));
}
@Override
public void disableSecretVersion(String secretId, String version) {
checkNotNull(secretId, "secretId");
checkNotNull(version, "version");
callSecretManager(
() -> csmClient.disableSecretVersion(SecretVersionName.of(project, secretId, version)));
}
@Override
public void destroySecretVersion(String secretId, String version) {
checkNotNull(secretId, "secretId");
@@ -14,12 +14,12 @@
package google.registry.privileges.secretmanager;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
import dagger.Component;
import dagger.Module;
import dagger.Provides;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.util.Retrier;
import google.registry.util.UtilsModule;
@@ -28,19 +28,16 @@ import javax.inject.Singleton;
/** Provides bindings for {@link SecretManagerClient}. */
@Module
public class SecretManagerModule {
private final String project;
public SecretManagerModule(String project) {
this.project = checkNotNull(project, "project");
}
public abstract class SecretManagerModule {
@Provides
@Singleton
SecretManagerClient provideSecretManagerClient(Retrier retrier) {
static SecretManagerClient provideSecretManagerClient(
@Config("projectId") String project, Retrier retrier) {
try {
return new SecretManagerClientImpl(project, SecretManagerServiceClient.create(), retrier);
SecretManagerServiceClient stub = SecretManagerServiceClient.create();
Runtime.getRuntime().addShutdownHook(new Thread(stub::close));
return new SecretManagerClientImpl(project, stub, retrier);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -0,0 +1,55 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.privileges.secretmanager;
import static avro.shaded.com.google.common.base.Preconditions.checkState;
import com.google.auto.value.AutoValue;
import java.util.List;
/**
* Contains the login name and password of a Cloud SQL user.
*
* <p>User must take care not to include the {@link #SEPARATOR} in property values.
*/
@AutoValue
public abstract class SqlCredential {
public static final Character SEPARATOR = ' ';
public abstract String login();
public abstract String password();
@Override
public final String toString() {
// Use Object.toString(), which does not show object data.
return super.toString();
}
public final String toFormattedString() {
return String.format("%s%c%s", login(), SEPARATOR, password());
}
public static SqlCredential fromFormattedString(String sqlCredential) {
List<String> items = com.google.common.base.Splitter.on(SEPARATOR).splitToList(sqlCredential);
checkState(items.size() == 2, "Invalid SqlCredential string.");
return of(items.get(0), items.get(1));
}
public static SqlCredential of(String login, String password) {
return new AutoValue_SqlCredential(login, password);
}
}
@@ -0,0 +1,118 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.privileges.secretmanager;
import com.google.cloud.secretmanager.v1.SecretVersionName;
import google.registry.config.RegistryConfig.Config;
import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
import java.util.Optional;
import javax.inject.Inject;
/**
* Storage of SQL users' login credentials, backed by Cloud Secret Manager.
*
* <p>A user's credential is stored with one level of indirection using two secret IDs: Each version
* of the <em>credential data</em> is stored as follows: its secret ID is determined by {@link
* #getCredentialDataSecretId(SqlUser, String dbInstance)}, and the value of each version is a
* {@link SqlCredential}, serialized using {@link SqlCredential#toFormattedString}. The 'live'
* version of the credential is saved under the 'live pointer' secret explained below.
*
* <p>The pointer to the 'live' version of the credential data is stored as follows: its secret ID
* is determined by {@link #getLiveLabelSecretId(SqlUser, String dbInstance)}; and the value of each
* version is a {@link SecretVersionName} in String form, pointing to a version of the credential
* data. Only the 'latest' version of this secret should be used. It is guaranteed to be valid.
*
* <p>The indirection in credential storage makes it easy to handle failures in the credential
* change process.
*/
public class SqlCredentialStore {
private final SecretManagerClient csmClient;
private final String dbInstance;
@Inject
SqlCredentialStore(
SecretManagerClient csmClient, @Config("cloudSqlDbInstanceName") String dbInstance) {
this.csmClient = csmClient;
this.dbInstance = dbInstance;
}
public SqlCredential getCredential(SqlUser user) {
SecretVersionName credentialName = getLiveCredentialSecretVersion(user);
return SqlCredential.fromFormattedString(
csmClient.getSecretData(
credentialName.getSecret(), Optional.of(credentialName.getSecretVersion())));
}
public void createOrUpdateCredential(SqlUser user, String password) {
SecretVersionName dataName = saveCredentialData(user, password);
saveLiveLabel(user, dataName);
}
public void deleteCredential(SqlUser user) {
try {
csmClient.deleteSecret(getCredentialDataSecretId(user, dbInstance));
} catch (NoSuchSecretResourceException e) {
// ok
}
try {
csmClient.deleteSecret(getLiveLabelSecretId(user, dbInstance));
} catch (NoSuchSecretResourceException e) {
// ok.
}
}
private void createSecretIfAbsent(String secretId) {
try {
csmClient.createSecret(secretId);
} catch (SecretAlreadyExistsException ignore) {
// Not a problem.
}
}
private SecretVersionName saveCredentialData(SqlUser user, String password) {
String credentialDataSecretId = getCredentialDataSecretId(user, dbInstance);
createSecretIfAbsent(credentialDataSecretId);
String credentialVersion =
csmClient.addSecretVersion(
credentialDataSecretId,
SqlCredential.of(createDatabaseLoginName(user), password).toFormattedString());
return SecretVersionName.of(csmClient.getProject(), credentialDataSecretId, credentialVersion);
}
private void saveLiveLabel(SqlUser user, SecretVersionName dataVersionName) {
String liveLabelSecretId = getLiveLabelSecretId(user, dbInstance);
createSecretIfAbsent(liveLabelSecretId);
csmClient.addSecretVersion(liveLabelSecretId, dataVersionName.toString());
}
private SecretVersionName getLiveCredentialSecretVersion(SqlUser user) {
return SecretVersionName.parse(
csmClient.getSecretData(getLiveLabelSecretId(user, dbInstance), Optional.empty()));
}
private static String getLiveLabelSecretId(SqlUser user, String dbInstance) {
return String.format("sql-cred-live-label-%s-%s", user.geUserName(), dbInstance);
}
private static String getCredentialDataSecretId(SqlUser user, String dbInstance) {
return String.format("sql-cred-data-%s-%s", user.geUserName(), dbInstance);
}
// WIP: when b/170230882 is complete, login will be versioned.
private static String createDatabaseLoginName(SqlUser user) {
return user.geUserName();
}
}
@@ -0,0 +1,62 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.privileges.secretmanager;
import com.google.common.base.Ascii;
/**
* SQL user information for privilege management purposes.
*
* <p>A {@link RobotUser} represents a software system accessing the database using its own
* credential. Robots are well known and enumerated in {@link RobotId}.
*/
public abstract class SqlUser {
private final UserType type;
private final String userName;
protected SqlUser(UserType type, String userName) {
this.type = type;
this.userName = userName;
}
public UserType getType() {
return type;
}
public String geUserName() {
return userName;
}
/** Cloud SQL user types. Please see class javadoc of {@link SqlUser} for more information. */
enum UserType {
// Work in progress. Human user will be added.
ROBOT
}
/** Enumerates the {@link RobotUser RobotUsers} in the system. */
public enum RobotId {
NOMULUS;
}
/** Information of a RobotUser for privilege management purposes. */
// Work in progress. Eventually will be provided based on configuration.
public static class RobotUser extends SqlUser {
public RobotUser(RobotId robot) {
super(UserType.ROBOT, Ascii.toLowerCase(robot.name()));
}
}
}
@@ -0,0 +1,73 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Ascii;
import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
import google.registry.privileges.secretmanager.SqlCredential;
import google.registry.privileges.secretmanager.SqlCredentialStore;
import google.registry.privileges.secretmanager.SqlUser;
import google.registry.privileges.secretmanager.SqlUser.RobotUser;
import google.registry.tools.params.PathParameter;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import javax.inject.Inject;
/**
* Command to get a Cloud SQL credential in the Secret Manager.
*
* <p>This command is a short-term tool that will be deprecated by the planned privilege server.
*/
@Parameters(separators = " =", commandDescription = "Get the Cloud SQL Credential for a given user")
public class GetSqlCredentialCommand implements Command {
@Inject SqlCredentialStore store;
@Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
private String user;
@Parameter(
names = {"-o", "--output"},
description = "Name of output file for key data.",
validateWith = PathParameter.OutputFile.class)
private Path outputPath = null;
@Inject
GetSqlCredentialCommand() {}
@Override
public void run() throws Exception {
SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
SqlCredential credential;
try {
credential = store.getCredential(sqlUser);
} catch (SecretManagerException e) {
System.out.println(e.getMessage());
return;
}
if (outputPath == null) {
System.out.printf("[%s]\n", credential.toFormattedString());
return;
}
try (FileOutputStream out = new FileOutputStream(outputPath.toFile())) {
out.write(credential.toFormattedString().getBytes(StandardCharsets.UTF_8));
}
}
}
@@ -78,6 +78,7 @@ public final class RegistryTool {
.put("get_routing_map", GetRoutingMapCommand.class)
.put("get_schema", GetSchemaCommand.class)
.put("get_schema_tree", GetSchemaTreeCommand.class)
.put("get_sql_credential", GetSqlCredentialCommand.class)
.put("get_tld", GetTldCommand.class)
.put("ghostryde", GhostrydeCommand.class)
.put("hash_certificate", HashCertificateCommand.class)
@@ -104,8 +105,10 @@ public final class RegistryTool {
.put("resave_entities", ResaveEntitiesCommand.class)
.put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class)
.put("resave_epp_resource", ResaveEppResourceCommand.class)
.put("save_sql_credential", SaveSqlCredentialCommand.class)
.put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class)
.put("set_num_instances", SetNumInstancesCommand.class)
.put("set_sql_replay_checkpoint", SetSqlReplayCheckpointCommand.class)
.put("setup_ote", SetupOteCommand.class)
.put("uniform_rapid_suspension", UniformRapidSuspensionCommand.class)
.put("unlock_domain", UnlockDomainCommand.class)
@@ -34,6 +34,7 @@ import google.registry.keyring.kms.KmsModule;
import google.registry.persistence.PersistenceModule;
import google.registry.persistence.PersistenceModule.NomulusToolJpaTm;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.privileges.secretmanager.SecretManagerModule;
import google.registry.rde.RdeModule;
import google.registry.request.Modules.DatastoreServiceModule;
import google.registry.request.Modules.Jackson2Module;
@@ -74,6 +75,7 @@ import javax.inject.Singleton;
PersistenceModule.class,
RdeModule.class,
RequestFactoryModule.class,
SecretManagerModule.class,
URLFetchServiceModule.class,
UrlFetchTransportModule.class,
UserServiceModule.class,
@@ -118,6 +120,8 @@ interface RegistryToolComponent {
void inject(GetOperationStatusCommand command);
void inject(GetSqlCredentialCommand command);
void inject(GhostrydeCommand command);
void inject(ImportDatastoreCommand command);
@@ -138,10 +142,14 @@ interface RegistryToolComponent {
void inject(RenewDomainCommand command);
void inject(SaveSqlCredentialCommand command);
void inject(SendEscrowReportToIcannCommand command);
void inject(SetNumInstancesCommand command);
void inject(SetSqlReplayCheckpointCommand command);
void inject(SetupOteCommand command);
void inject(UnlockDomainCommand command);
@@ -0,0 +1,69 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Ascii;
import google.registry.privileges.secretmanager.SqlCredentialStore;
import google.registry.privileges.secretmanager.SqlUser;
import google.registry.privileges.secretmanager.SqlUser.RobotUser;
import google.registry.tools.params.PathParameter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.inject.Inject;
/**
* Command to create or update a Cloud SQL credential in the Secret Manager.
*
* <p>This command is a short-term tool that will be deprecated by the planned privilege server.
*/
@Parameters(
separators = " =",
commandDescription = "Create or update the Cloud SQL Credential for a given user")
public class SaveSqlCredentialCommand implements Command {
@Inject SqlCredentialStore store;
@Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
private String user;
@Parameter(
names = {"--input"},
description =
"Name of input file for the password. If absent, command will prompt for "
+ "password in console.",
validateWith = PathParameter.InputFile.class)
private Path inputPath = null;
@Inject
SaveSqlCredentialCommand() {}
@Override
public void run() throws Exception {
String password = getPassword();
SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
store.createOrUpdateCredential(sqlUser, password);
System.out.printf("\nDone:[%s]\n", password);
}
private String getPassword() throws Exception {
if (inputPath != null) {
return Files.readAllLines(inputPath, StandardCharsets.UTF_8).get(0);
}
return System.console().readLine("Please enter the password: ").trim();
}
}
@@ -0,0 +1,48 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.collect.Iterables;
import google.registry.schema.replay.SqlReplayCheckpoint;
import java.util.List;
import org.joda.time.DateTime;
/** Command to set {@link SqlReplayCheckpoint} to a particular, post-initial-population time. */
@Parameters(separators = " =", commandDescription = "Set SqlReplayCheckpoint to a particular time")
public class SetSqlReplayCheckpointCommand extends ConfirmingCommand
implements CommandWithRemoteApi {
@Parameter(description = "Time to which SqlReplayCheckpoint will be set", required = true)
List<DateTime> mainParameters;
@Override
protected String prompt() {
checkArgument(mainParameters.size() == 1, "Must provide exactly one DateTime to set");
return String.format(
"Set SqlReplayCheckpoint to %s?", Iterables.getOnlyElement(mainParameters));
}
@Override
protected String execute() {
DateTime dateTime = Iterables.getOnlyElement(mainParameters);
jpaTm().transact(() -> SqlReplayCheckpoint.set(dateTime));
return String.format("Set SqlReplayCheckpoint time to %s", dateTime);
}
}
@@ -17,13 +17,17 @@ package google.registry.model.host;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.model.EppResourceUtils.loadByForeignKey;
import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.testing.DatabaseHelper.cloneAndSetAutoTimestamps;
import static google.registry.testing.DatabaseHelper.createTld;
import static google.registry.testing.DatabaseHelper.newDomainBase;
import static google.registry.testing.DatabaseHelper.persistNewRegistrars;
import static google.registry.testing.DatabaseHelper.persistResource;
import static google.registry.testing.HostResourceSubject.assertAboutHosts;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.InetAddresses;
import google.registry.model.EntityTestCase;
@@ -32,11 +36,14 @@ import google.registry.model.eppcommon.StatusValue;
import google.registry.model.eppcommon.Trid;
import google.registry.model.transfer.DomainTransferData;
import google.registry.model.transfer.TransferStatus;
import google.registry.testing.DualDatabaseTest;
import google.registry.testing.TestOfyAndSql;
import google.registry.testing.TestOfyOnly;
import org.joda.time.DateTime;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/** Unit tests for {@link HostResource}. */
@DualDatabaseTest
class HostResourceTest extends EntityTestCase {
private final DateTime day3 = fakeClock.nowUtc();
@@ -49,6 +56,7 @@ class HostResourceTest extends EntityTestCase {
@BeforeEach
void setUp() {
createTld("com");
persistNewRegistrars("gaining", "losing", "thisRegistrar", "thatRegistrar");
// Set up a new persisted registrar entity.
domain =
persistResource(
@@ -71,9 +79,9 @@ class HostResourceTest extends EntityTestCase {
new HostResource.Builder()
.setRepoId("DEADBEEF-COM")
.setHostName("ns1.example.com")
.setCreationClientId("a registrar")
.setCreationClientId("thisRegistrar")
.setLastEppUpdateTime(fakeClock.nowUtc())
.setLastEppUpdateClientId("another registrar")
.setLastEppUpdateClientId("thatRegistrar")
.setLastTransferTime(fakeClock.nowUtc())
.setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
.setStatusValues(ImmutableSet.of(StatusValue.OK))
@@ -81,13 +89,22 @@ class HostResourceTest extends EntityTestCase {
.build()));
}
@Test
@TestOfyAndSql
void testPersistence() {
HostResource newHost = host.asBuilder().setRepoId("NEWHOST").build();
tm().transact(() -> tm().insert(newHost));
assertThat(ImmutableList.of(tm().transact(() -> tm().load(newHost.createVKey()))))
.comparingElementsUsing(immutableObjectCorrespondence("revisions"))
.containsExactly(newHost);
}
@TestOfyOnly
void testLoadingByForeignKey() {
assertThat(loadByForeignKey(HostResource.class, host.getForeignKey(), fakeClock.nowUtc()))
.hasValue(host);
}
@Test
@TestOfyOnly
void testIndexing() throws Exception {
// Clone it and save it before running the indexing test so that its transferData fields are
// populated from the superordinate domain.
@@ -100,7 +117,7 @@ class HostResourceTest extends EntityTestCase {
"currentSponsorClientId");
}
@Test
@TestOfyAndSql
void testEmptyStringsBecomeNull() {
assertThat(
new HostResource.Builder()
@@ -122,7 +139,7 @@ class HostResourceTest extends EntityTestCase {
.isNotNull();
}
@Test
@TestOfyAndSql
void testEmptySetsBecomeNull() {
assertThat(new HostResource.Builder().setInetAddresses(null).build().inetAddresses).isNull();
assertThat(new HostResource.Builder().setInetAddresses(ImmutableSet.of()).build().inetAddresses)
@@ -135,7 +152,7 @@ class HostResourceTest extends EntityTestCase {
.isNotNull();
}
@Test
@TestOfyAndSql
void testImplicitStatusValues() {
// OK is implicit if there's no other statuses.
assertAboutHosts()
@@ -157,13 +174,13 @@ class HostResourceTest extends EntityTestCase {
.hasExactlyStatusValues(StatusValue.CLIENT_HOLD);
}
@Test
@TestOfyAndSql
void testToHydratedString_notCircular() {
// If there are circular references, this will overflow the stack.
host.toHydratedString();
}
@Test
@TestOfyAndSql
void testFailure_uppercaseHostName() {
IllegalArgumentException thrown =
assertThrows(
@@ -173,7 +190,7 @@ class HostResourceTest extends EntityTestCase {
.contains("Host name must be in puny-coded, lower-case form");
}
@Test
@TestOfyAndSql
void testFailure_utf8HostName() {
IllegalArgumentException thrown =
assertThrows(
@@ -183,14 +200,14 @@ class HostResourceTest extends EntityTestCase {
.contains("Host name must be in puny-coded, lower-case form");
}
@Test
@TestOfyAndSql
void testComputeLastTransferTime_hostNeverSwitchedDomains_domainWasNeverTransferred() {
domain = domain.asBuilder().setLastTransferTime(null).build();
host = host.asBuilder().setLastTransferTime(null).setLastSuperordinateChange(null).build();
assertThat(host.computeLastTransferTime(domain)).isNull();
}
@Test
@TestOfyAndSql
void testComputeLastTransferTime_hostNeverSwitchedDomains_domainWasTransferred() {
// Host was created on Day 1.
// Domain was transferred on Day 2.
@@ -205,7 +222,7 @@ class HostResourceTest extends EntityTestCase {
assertThat(host.computeLastTransferTime(domain)).isEqualTo(day2);
}
@Test
@TestOfyAndSql
void testComputeLastTransferTime_hostCreatedAfterDomainWasTransferred() {
// Domain was transferred on Day 1.
// Host was created subordinate to domain on Day 2.
@@ -217,9 +234,9 @@ class HostResourceTest extends EntityTestCase {
.setCreationTime(day2)
.setRepoId("DEADBEEF-COM")
.setHostName("ns1.example.com")
.setCreationClientId("a registrar")
.setCreationClientId("thisRegistrar")
.setLastEppUpdateTime(fakeClock.nowUtc())
.setLastEppUpdateClientId("another registrar")
.setLastEppUpdateClientId("thatRegistrar")
.setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
.setStatusValues(ImmutableSet.of(StatusValue.OK))
.setSuperordinateDomain(domain.createVKey())
@@ -227,7 +244,7 @@ class HostResourceTest extends EntityTestCase {
assertThat(host.computeLastTransferTime(domain)).isNull();
}
@Test
@TestOfyAndSql
void testComputeLastTransferTime_hostWasTransferred_domainWasNeverTransferred() {
// Host was transferred on Day 1.
// Host was made subordinate to domain on Day 2.
@@ -237,7 +254,7 @@ class HostResourceTest extends EntityTestCase {
assertThat(host.computeLastTransferTime(domain)).isEqualTo(day1);
}
@Test
@TestOfyAndSql
void testComputeLastTransferTime_domainWasTransferredBeforeHostBecameSubordinate() {
// Host was transferred on Day 1.
// Domain was transferred on Day 2.
@@ -247,7 +264,7 @@ class HostResourceTest extends EntityTestCase {
assertThat(host.computeLastTransferTime(domain)).isEqualTo(day1);
}
@Test
@TestOfyAndSql
void testComputeLastTransferTime_domainWasTransferredAfterHostBecameSubordinate() {
// Host was transferred on Day 1.
// Host was made subordinate to domain on Day 2.
@@ -32,6 +32,11 @@ public class FakeSecretManagerClient implements SecretManagerClient {
@Inject
FakeSecretManagerClient() {}
@Override
public String getProject() {
return "fake_project";
}
@Override
public void createSecret(String secretId) {
checkNotNull(secretId, "secretId");
@@ -41,6 +46,12 @@ public class FakeSecretManagerClient implements SecretManagerClient {
secrets.put(secretId, new SecretEntry(secretId));
}
@Override
public boolean secretExists(String secretId) {
checkNotNull(secretId, "secretId");
return secrets.containsKey(secretId);
}
@Override
public Iterable<String> listSecrets() {
return ImmutableSet.copyOf(secrets.keySet());
@@ -78,6 +89,28 @@ public class FakeSecretManagerClient implements SecretManagerClient {
return secretEntry.getVersion(version).getData();
}
@Override
public void enableSecretVersion(String secretId, String version) {
checkNotNull(secretId, "secretId");
checkNotNull(version, "version");
SecretEntry secretEntry = secrets.get(secretId);
if (secretEntry == null) {
throw new NoSuchSecretResourceException(null);
}
secretEntry.enableVersion(version);
}
@Override
public void disableSecretVersion(String secretId, String version) {
checkNotNull(secretId, "secretId");
checkNotNull(version, "version");
SecretEntry secretEntry = secrets.get(secretId);
if (secretEntry == null) {
throw new NoSuchSecretResourceException(null);
}
secretEntry.disableVersion(version);
}
@Override
public void destroySecretVersion(String secretId, String version) {
checkNotNull(secretId, "secretId");
@@ -118,6 +151,20 @@ public class FakeSecretManagerClient implements SecretManagerClient {
return state;
}
void enable() {
if (state.equals(State.DESTROYED)) {
throw new SecretManagerException(null);
}
state = State.ENABLED;
}
void disable() {
if (state.equals(State.DESTROYED)) {
throw new SecretManagerException(null);
}
state = State.DISABLED;
}
void destroy() {
data = null;
state = State.DESTROYED;
@@ -145,6 +192,8 @@ public class FakeSecretManagerClient implements SecretManagerClient {
return versions.get(index);
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid version " + version.get());
} catch (ArrayIndexOutOfBoundsException e) {
throw new NoSuchSecretResourceException(null);
}
}
@@ -156,15 +205,16 @@ public class FakeSecretManagerClient implements SecretManagerClient {
return builder.build();
}
void enableVersion(String version) {
getVersion(Optional.of(version)).enable();
}
void disableVersion(String version) {
getVersion(Optional.of(version)).disable();
}
void destroyVersion(String version) {
try {
int index = Integer.valueOf(version);
versions.get(index).destroy();
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid version " + version);
} catch (ArrayIndexOutOfBoundsException e) {
throw new NoSuchSecretResourceException(null);
}
getVersion(Optional.of(version)).destroy();
}
}
}
@@ -18,12 +18,17 @@ import static com.google.common.truth.Truth.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import com.google.cloud.secretmanager.v1.SecretVersion.State;
import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
import google.registry.util.Retrier;
import google.registry.util.SystemSleeper;
import java.io.IOException;
import java.util.Optional;
import java.util.UUID;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/**
@@ -43,52 +48,50 @@ public class SecretManagerClientTest {
private static final String SECRET_ID_PREFIX = "TEST_" + UUID.randomUUID() + "_";
// Used for unique secret id generation.
private static int seqno = 0;
private static SecretManagerClient secretManagerClient;
private static boolean isUnitTest = true;
private static String nextSecretId() {
return SECRET_ID_PREFIX + seqno++;
}
private String secretId;
@BeforeAll
static void beforeAll() {
String environmentName = System.getProperty("test.gcp_integration.env");
if (environmentName != null) {
secretManagerClient =
DaggerSecretManagerModule_SecretManagerComponent.builder()
.secretManagerModule(
new SecretManagerModule(String.format("domain-registry-%s", environmentName)))
.build()
.secretManagerClient();
SecretManagerModule.provideSecretManagerClient(
String.format("domain-registry-%s", environmentName),
new Retrier(new SystemSleeper(), 1));
isUnitTest = false;
} else {
secretManagerClient = new FakeSecretManagerClient();
}
}
@AfterAll
static void afterAll() {
@BeforeEach
void beforeEach() {
secretId = SECRET_ID_PREFIX + seqno++;
}
@AfterEach
void afterEach() throws IOException {
if (isUnitTest) {
return;
}
for (String secretId : secretManagerClient.listSecrets()) {
if (secretId.startsWith(SECRET_ID_PREFIX)) {
secretManagerClient.deleteSecret(secretId);
}
try {
secretManagerClient.deleteSecret(secretId);
} catch (NoSuchSecretResourceException e) {
// deleteSecret() deleted it already.
}
}
@Test
void createSecret_success() {
String secretId = nextSecretId();
secretManagerClient.createSecret(secretId);
assertThat(secretManagerClient.listSecrets()).contains(secretId);
}
@Test
void createSecret_duplicate() {
String secretId = nextSecretId();
secretManagerClient.createSecret(secretId);
assertThrows(
SecretAlreadyExistsException.class, () -> secretManagerClient.createSecret(secretId));
@@ -96,16 +99,25 @@ public class SecretManagerClientTest {
@Test
void addSecretVersion() {
String secretId = nextSecretId();
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED))
.containsExactly(version);
}
@Test
void secretExists_true() {
secretManagerClient.createSecret(secretId);
assertThat(secretManagerClient.secretExists(secretId)).isTrue();
}
@Test
void secretExists_False() {
assertThat(secretManagerClient.secretExists(secretId)).isFalse();
}
@Test
void getSecretData_byVersion() {
String secretId = nextSecretId();
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
assertThat(secretManagerClient.getSecretData(secretId, Optional.of(version)))
@@ -114,15 +126,70 @@ public class SecretManagerClientTest {
@Test
void getSecretData_latestVersion() {
String secretId = nextSecretId();
secretManagerClient.createSecret(secretId);
secretManagerClient.addSecretVersion(secretId, "mydata");
assertThat(secretManagerClient.getSecretData(secretId, Optional.empty())).isEqualTo("mydata");
}
@Test
void disableSecretVersion() {
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
secretManagerClient.disableSecretVersion(secretId, version);
assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
}
@Test
void disableSecretVersion_ignoreAlreadyDisabled() {
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
secretManagerClient.disableSecretVersion(secretId, version);
assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
secretManagerClient.disableSecretVersion(secretId, version);
}
@Test
void disableSecretVersion_destroyed() {
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
secretManagerClient.destroySecretVersion(secretId, version);
assertThat(secretManagerClient.listSecretVersions(secretId, State.DESTROYED)).contains(version);
assertThrows(
SecretManagerException.class,
() -> secretManagerClient.disableSecretVersion(secretId, version));
}
@Test
void enableSecretVersion_ignoreAlreadyEnabled() {
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED)).contains(version);
secretManagerClient.enableSecretVersion(secretId, version);
}
@Test
void enableSecretVersion() {
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
secretManagerClient.disableSecretVersion(secretId, version);
assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
secretManagerClient.enableSecretVersion(secretId, version);
assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED)).contains(version);
}
@Test
void enableSecretVersion_destroyed() {
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
secretManagerClient.destroySecretVersion(secretId, version);
assertThat(secretManagerClient.listSecretVersions(secretId, State.DESTROYED)).contains(version);
assertThrows(
SecretManagerException.class,
() -> secretManagerClient.enableSecretVersion(secretId, version));
}
@Test
void destroySecretVersion() {
String secretId = nextSecretId();
secretManagerClient.createSecret(secretId);
String version = secretManagerClient.addSecretVersion(secretId, "mydata");
secretManagerClient.destroySecretVersion(secretId, version);
@@ -134,7 +201,6 @@ public class SecretManagerClientTest {
@Test
void deleteSecret() {
String secretId = nextSecretId();
secretManagerClient.createSecret(secretId);
assertThat(secretManagerClient.listSecrets()).contains(secretId);
secretManagerClient.deleteSecret(secretId);
@@ -0,0 +1,63 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.privileges.secretmanager;
import static com.google.common.truth.Truth.assertThat;
import com.google.cloud.secretmanager.v1.SecretVersionName;
import google.registry.privileges.secretmanager.SqlUser.RobotId;
import google.registry.privileges.secretmanager.SqlUser.RobotUser;
import java.util.Optional;
import org.junit.jupiter.api.Test;
/** Unit tests for {@link SqlCredentialStore}. */
public class SqlCredentialStoreTest {
private final SecretManagerClient client = new FakeSecretManagerClient();
private final SqlCredentialStore credentialStore = new SqlCredentialStore(client, "db");
private SqlUser user = new RobotUser(RobotId.NOMULUS);
@Test
void createSecret() {
credentialStore.createOrUpdateCredential(user, "password");
assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isTrue();
assertThat(
SecretVersionName.parse(
client.getSecretData("sql-cred-live-label-nomulus-db", Optional.empty()))
.getSecret())
.isEqualTo("sql-cred-data-nomulus-db");
assertThat(client.secretExists("sql-cred-data-nomulus-db")).isTrue();
assertThat(client.getSecretData("sql-cred-data-nomulus-db", Optional.empty()))
.isEqualTo("nomulus password");
}
@Test
void getCredential() {
credentialStore.createOrUpdateCredential(user, "password");
SqlCredential credential = credentialStore.getCredential(user);
assertThat(credential.login()).isEqualTo("nomulus");
assertThat(credential.password()).isEqualTo("password");
}
@Test
void deleteCredential() {
credentialStore.createOrUpdateCredential(user, "password");
assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isTrue();
assertThat(client.secretExists("sql-cred-data-nomulus-db")).isTrue();
credentialStore.deleteCredential(user);
assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isFalse();
assertThat(client.secretExists("sql-cred-data-nomulus-db")).isFalse();
}
}
@@ -750,6 +750,20 @@ public class DatabaseHelper {
.build());
}
/** Persists and returns a {@link Registrar} with the specified registrarId. */
public static Registrar persistNewRegistrar(String registrarId) {
return persistNewRegistrar(registrarId, registrarId + " name", Registrar.Type.REAL, 100L);
}
/** Persists and returns a list of {@link Registrar}s with the specified registrarIds. */
public static ImmutableList<Registrar> persistNewRegistrars(String... registrarIds) {
ImmutableList.Builder<Registrar> newRegistrars = new ImmutableList.Builder<>();
for (String registrarId : registrarIds) {
newRegistrars.add(persistNewRegistrar(registrarId));
}
return newRegistrars.build();
}
private static Iterable<BillingEvent> getBillingEvents() {
return transactIfJpaTm(
() ->
@@ -0,0 +1,42 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static org.junit.Assert.assertThrows;
import google.registry.schema.replay.SqlReplayCheckpoint;
import org.joda.time.DateTime;
import org.junit.jupiter.api.Test;
class SetSqlReplayCheckpointCommandTest extends CommandTestCase<SetSqlReplayCheckpointCommand> {
@Test
void testSuccess() throws Exception {
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(START_OF_TIME);
DateTime timeToSet = DateTime.parse("2000-06-06T22:00:00.0Z");
runCommandForced(timeToSet.toString());
assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(timeToSet);
}
@Test
void testFailure_multipleParams() throws Exception {
DateTime one = DateTime.parse("2000-06-06T22:00:00.0Z");
DateTime two = DateTime.parse("2001-06-06T22:00:00.0Z");
assertThrows(IllegalArgumentException.class, () -> runCommand(one.toString(), two.toString()));
}
}