mirror of
https://github.com/google/nomulus
synced 2026-05-19 06:11:49 +00:00
Compare commits
5 Commits
nomulus-20
...
nomulus-20
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8a06ef09c0 | ||
|
|
9e0368b77c | ||
|
|
05c45da07a | ||
|
|
365c5da942 | ||
|
|
2cc2571375 |
@@ -99,6 +99,12 @@ PRESUBMITS = {
|
||||
"System.(out|err).println is only allowed in tools/ packages. Please "
|
||||
"use a logger instead.",
|
||||
|
||||
# PostgreSQLContainer instantiation must specify docker tag
|
||||
PresubmitCheck(
|
||||
r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
|
||||
"java", {}):
|
||||
"PostgreSQLContainer instantiation must specify docker tag.",
|
||||
|
||||
# Various Soy linting checks
|
||||
PresubmitCheck(
|
||||
r".* (/\*)?\* {?@param ",
|
||||
|
||||
@@ -243,6 +243,8 @@ dependencies {
|
||||
// Known issue: nebula-lint misses inherited dependency.
|
||||
compile project(':third_party')
|
||||
compile project(':util')
|
||||
// Import NomulusPostreSql from ':db' for compile but exclude dependencies.
|
||||
compile project(path: ':db', configuration: 'compileApi')
|
||||
testRuntime project(':db')
|
||||
|
||||
// Include auto-value in compile until nebula-lint understands
|
||||
|
||||
@@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.net.InternetDomainName;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
import java.util.Optional;
|
||||
|
||||
@@ -80,6 +81,15 @@ public final class Registries {
|
||||
return ImmutableSet.copyOf(filterValues(cache.get(), equalTo(type)).keySet());
|
||||
}
|
||||
|
||||
/** Returns the Registry entities themselves of the given type loaded fresh from Datastore. */
|
||||
public static ImmutableSet<Registry> getTldEntitiesOfType(TldType type) {
|
||||
ImmutableSet<Key<Registry>> keys =
|
||||
filterValues(cache.get(), equalTo(type)).keySet().stream()
|
||||
.map(tld -> Key.create(getCrossTldKey(), Registry.class, tld))
|
||||
.collect(toImmutableSet());
|
||||
return ImmutableSet.copyOf(tm().doTransactionless(() -> ofy().load().keys(keys).values()));
|
||||
}
|
||||
|
||||
/** Pass-through check that the specified TLD exists, otherwise throw an IAE. */
|
||||
public static String assertTldExists(String tld) {
|
||||
checkArgument(
|
||||
|
||||
@@ -14,16 +14,13 @@
|
||||
|
||||
package google.registry.model.transaction;
|
||||
|
||||
import static google.registry.config.RegistryEnvironment.ALPHA;
|
||||
import static google.registry.config.RegistryEnvironment.CRASH;
|
||||
import static google.registry.config.RegistryEnvironment.SANDBOX;
|
||||
|
||||
import com.google.appengine.api.utils.SystemProperty;
|
||||
import com.google.appengine.api.utils.SystemProperty.Environment.Value;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.ofy.DatastoreTransactionManager;
|
||||
import google.registry.persistence.DaggerPersistenceComponent;
|
||||
import google.registry.tools.RegistryToolEnvironment;
|
||||
|
||||
/** Factory class to create {@link TransactionManager} instance. */
|
||||
// TODO: Rename this to PersistenceFactory and move to persistence package.
|
||||
@@ -35,8 +32,11 @@ public class TransactionManagerFactory {
|
||||
private TransactionManagerFactory() {}
|
||||
|
||||
private static JpaTransactionManager createJpaTransactionManager() {
|
||||
if (shouldEnableJpaTm() && isInAppEngine()) {
|
||||
if (isInAppEngine()) {
|
||||
return DaggerPersistenceComponent.create().appEngineJpaTransactionManager();
|
||||
} else if (RegistryToolEnvironment.isInRegistryTool()
|
||||
&& RegistryToolEnvironment.isJpaTmEnabled()) {
|
||||
return DaggerPersistenceComponent.create().nomulusToolJpaTransactionManager();
|
||||
} else {
|
||||
return DummyJpaTransactionManager.create();
|
||||
}
|
||||
@@ -49,23 +49,6 @@ public class TransactionManagerFactory {
|
||||
return new DatastoreTransactionManager(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets jpaTm to the implementation for Nomulus tool. Note that this method should be only used by
|
||||
* {@link google.registry.tools.RegistryCli} to initialize jpaTm.
|
||||
*/
|
||||
public static void initForTool() {
|
||||
if (shouldEnableJpaTm()) {
|
||||
jpaTm = DaggerPersistenceComponent.create().nomulusToolJpaTransactionManager();
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(shicong): Enable JpaTm for all environments and remove this function
|
||||
private static boolean shouldEnableJpaTm() {
|
||||
return RegistryEnvironment.get() == ALPHA
|
||||
|| RegistryEnvironment.get() == CRASH
|
||||
|| RegistryEnvironment.get() == SANDBOX;
|
||||
}
|
||||
|
||||
/**
|
||||
* This function uses App Engine API to determine if the current runtime environment is App
|
||||
* Engine.
|
||||
|
||||
@@ -20,6 +20,7 @@ import com.beust.jcommander.Parameter;
|
||||
import com.beust.jcommander.Parameters;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import google.registry.persistence.HibernateSchemaExporter;
|
||||
import google.registry.persistence.NomulusPostgreSql;
|
||||
import google.registry.persistence.PersistenceXmlUtility;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
@@ -83,7 +84,7 @@ public class GenerateSqlSchemaCommand implements Command {
|
||||
|
||||
// Start the container and store the address information.
|
||||
postgresContainer =
|
||||
new PostgreSQLContainer()
|
||||
new PostgreSQLContainer(NomulusPostgreSql.getDockerTag())
|
||||
.withDatabaseName(DB_NAME)
|
||||
.withUsername(DB_USERNAME)
|
||||
.withPassword(DB_PASSWORD);
|
||||
|
||||
@@ -31,7 +31,6 @@ import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Iterables;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.ofy.ObjectifyService;
|
||||
import google.registry.model.transaction.TransactionManagerFactory;
|
||||
import google.registry.tools.AuthModule.LoginRequiredException;
|
||||
import google.registry.tools.params.ParameterFactory;
|
||||
import java.io.ByteArrayInputStream;
|
||||
@@ -237,7 +236,7 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
|
||||
}
|
||||
|
||||
if (command instanceof CommandWithCloudSql) {
|
||||
TransactionManagerFactory.initForTool();
|
||||
RegistryToolEnvironment.get().enableJpaTm();
|
||||
}
|
||||
|
||||
command.run();
|
||||
|
||||
@@ -25,7 +25,7 @@ import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.config.SystemPropertySetter;
|
||||
|
||||
/** Enum of production environments, used for the {@code --environment} flag. */
|
||||
enum RegistryToolEnvironment {
|
||||
public enum RegistryToolEnvironment {
|
||||
PRODUCTION(RegistryEnvironment.PRODUCTION),
|
||||
ALPHA(RegistryEnvironment.ALPHA),
|
||||
CRASH(RegistryEnvironment.CRASH),
|
||||
@@ -39,6 +39,7 @@ enum RegistryToolEnvironment {
|
||||
|
||||
private static final ImmutableList<String> FLAGS = ImmutableList.of("-e", "--environment");
|
||||
private static RegistryToolEnvironment instance;
|
||||
private static boolean isJpaTmEnabled = false;
|
||||
private final RegistryEnvironment actualEnvironment;
|
||||
private final ImmutableMap<String, String> extraProperties;
|
||||
|
||||
@@ -72,7 +73,7 @@ enum RegistryToolEnvironment {
|
||||
*
|
||||
* <p>This should be called after {@link #parseFromArgs(String[])}.
|
||||
*/
|
||||
static RegistryToolEnvironment get() {
|
||||
public static RegistryToolEnvironment get() {
|
||||
checkState(instance != null, "No RegistryToolEnvironment has been set up");
|
||||
return instance;
|
||||
}
|
||||
@@ -98,6 +99,26 @@ enum RegistryToolEnvironment {
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns true if the RegistryToolEnvironment is set up. */
|
||||
public static boolean isInRegistryTool() {
|
||||
return instance != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the flag to indicate that the running command needs JpaTransactionManager to be enabled.
|
||||
*/
|
||||
public static void enableJpaTm() {
|
||||
isJpaTmEnabled = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the JpaTransactionManager is enabled. Note that JpaTm is actually enabled in
|
||||
* {@link google.registry.model.transaction.TransactionManagerFactory} by reading this flag.
|
||||
*/
|
||||
public static boolean isJpaTmEnabled() {
|
||||
return isJpaTmEnabled;
|
||||
}
|
||||
|
||||
/** Extracts value from command-line arguments associated with any {@code flags}. */
|
||||
private static String getFlagValue(String[] args, Iterable<String> flags) {
|
||||
for (String flag : flags) {
|
||||
|
||||
@@ -17,9 +17,12 @@ package google.registry.model.registry;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth8.assertThat;
|
||||
import static google.registry.testing.DatastoreHelper.createTlds;
|
||||
import static google.registry.testing.DatastoreHelper.newRegistry;
|
||||
import static google.registry.testing.DatastoreHelper.persistResource;
|
||||
import static google.registry.testing.JUnitBackports.assertThrows;
|
||||
|
||||
import com.google.common.net.InternetDomainName;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
import google.registry.testing.AppEngineRule;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
@@ -32,6 +35,7 @@ public class RegistriesTest {
|
||||
|
||||
@Rule
|
||||
public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
|
||||
|
||||
private void initTestTlds() {
|
||||
createTlds("foo", "a.b.c"); // Test a multipart tld.
|
||||
}
|
||||
@@ -42,6 +46,16 @@ public class RegistriesTest {
|
||||
assertThat(Registries.getTlds()).containsExactly("foo", "a.b.c");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_getTldEntities() {
|
||||
initTestTlds();
|
||||
persistResource(newRegistry("testtld", "TESTTLD").asBuilder().setTldType(TldType.TEST).build());
|
||||
assertThat(Registries.getTldEntitiesOfType(TldType.REAL))
|
||||
.containsExactly(Registry.get("foo"), Registry.get("a.b.c"));
|
||||
assertThat(Registries.getTldEntitiesOfType(TldType.TEST))
|
||||
.containsExactly(Registry.get("testtld"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTlds_withNoRegistriesPersisted_returnsEmptySet() {
|
||||
assertThat(Registries.getTlds()).isEmpty();
|
||||
|
||||
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.io.Resources;
|
||||
import google.registry.persistence.HibernateSchemaExporter;
|
||||
import google.registry.persistence.NomulusPostgreSql;
|
||||
import google.registry.persistence.PersistenceModule;
|
||||
import google.registry.persistence.PersistenceXmlUtility;
|
||||
import google.registry.testing.FakeClock;
|
||||
@@ -74,6 +75,9 @@ public class JpaTransactionManagerRule extends ExternalResource {
|
||||
private final ImmutableMap userProperties;
|
||||
|
||||
private static final JdbcDatabaseContainer database = create();
|
||||
private static final long ACTIVE_CONNECTIONS_BASELINE =
|
||||
getActiveConnectionCountByUser(database.getUsername());
|
||||
;
|
||||
private static final HibernateSchemaExporter exporter =
|
||||
HibernateSchemaExporter.create(
|
||||
database.getJdbcUrl(), database.getUsername(), database.getPassword());
|
||||
@@ -90,7 +94,9 @@ public class JpaTransactionManagerRule extends ExternalResource {
|
||||
}
|
||||
|
||||
private static JdbcDatabaseContainer create() {
|
||||
PostgreSQLContainer container = new PostgreSQLContainer().withDatabaseName(MANAGEMENT_DB_NAME);
|
||||
PostgreSQLContainer container =
|
||||
new PostgreSQLContainer(NomulusPostgreSql.getDockerTag())
|
||||
.withDatabaseName(MANAGEMENT_DB_NAME);
|
||||
container.start();
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(() -> container.close()));
|
||||
return container;
|
||||
@@ -140,25 +146,26 @@ public class JpaTransactionManagerRule extends ExternalResource {
|
||||
assertNormalActiveConnection();
|
||||
}
|
||||
|
||||
private static long getActiveConnectionCountByUser(String userName) {
|
||||
try (Connection conn = createConnection(POSTGRES_DB_NAME);
|
||||
Statement statement = conn.createStatement()) {
|
||||
ResultSet rs =
|
||||
statement.executeQuery(
|
||||
"SELECT COUNT(1) FROM pg_stat_activity WHERE usename = '" + userName + "'");
|
||||
rs.next();
|
||||
return rs.getLong(1);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This function throws exception if it detects connection leak by checking the metadata table
|
||||
* pg_stat_activity.
|
||||
*/
|
||||
private void assertNormalActiveConnection() {
|
||||
try (Connection conn = createConnection(POSTGRES_DB_NAME);
|
||||
Statement statement = conn.createStatement()) {
|
||||
ResultSet rs =
|
||||
statement.executeQuery(
|
||||
"SELECT COUNT(1) FROM pg_stat_activity WHERE usename = '"
|
||||
+ database.getUsername()
|
||||
+ "'");
|
||||
rs.next();
|
||||
long activeConns = rs.getLong(1);
|
||||
// There should be only 1 active connection which is executing this query
|
||||
assertThat(activeConns).isEqualTo(1L);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
assertThat(getActiveConnectionCountByUser(database.getUsername()))
|
||||
.isEqualTo(ACTIVE_CONNECTIONS_BASELINE);
|
||||
}
|
||||
|
||||
private static String readSqlInClassPath(String sqlScriptPath) {
|
||||
@@ -178,7 +185,7 @@ public class JpaTransactionManagerRule extends ExternalResource {
|
||||
}
|
||||
}
|
||||
|
||||
private String getJdbcUrlFor(String dbName) {
|
||||
private static String getJdbcUrlFor(String dbName) {
|
||||
// Disable Postgres driver use of java.util.logging to reduce noise at startup time
|
||||
return "jdbc:postgresql://"
|
||||
+ database.getContainerIpAddress()
|
||||
@@ -189,7 +196,7 @@ public class JpaTransactionManagerRule extends ExternalResource {
|
||||
+ "?loggerLevel=OFF";
|
||||
}
|
||||
|
||||
private Connection createConnection(String dbName) {
|
||||
private static Connection createConnection(String dbName) {
|
||||
final Properties info = new Properties();
|
||||
info.put("user", database.getUsername());
|
||||
info.put("password", database.getPassword());
|
||||
|
||||
@@ -36,7 +36,10 @@ import org.testcontainers.containers.PostgreSQLContainer;
|
||||
/** Unit tests for {@link HibernateSchemaExporter}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class HibernateSchemaExporterTest {
|
||||
@ClassRule public static final PostgreSQLContainer database = new PostgreSQLContainer();
|
||||
@ClassRule
|
||||
public static final PostgreSQLContainer database =
|
||||
new PostgreSQLContainer(NomulusPostgreSql.getDockerTag());
|
||||
|
||||
private static HibernateSchemaExporter exporter;
|
||||
|
||||
@Rule public final TemporaryFolder tempFolder = new TemporaryFolder();
|
||||
|
||||
@@ -29,7 +29,8 @@ import org.testcontainers.containers.PostgreSQLContainer;
|
||||
/** Unit tests for {@link PersistenceModule}. */
|
||||
@RunWith(JUnit4.class)
|
||||
public class PersistenceModuleTest {
|
||||
@Rule public PostgreSQLContainer database = new PostgreSQLContainer();
|
||||
@Rule
|
||||
public PostgreSQLContainer database = new PostgreSQLContainer(NomulusPostgreSql.getDockerTag());
|
||||
|
||||
private EntityManagerFactory emf;
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.persistence.NomulusPostgreSql;
|
||||
import java.io.File;
|
||||
import org.junit.Before;
|
||||
import org.junit.ClassRule;
|
||||
@@ -37,8 +38,9 @@ public class GenerateSqlSchemaCommandTest extends CommandTestCase<GenerateSqlSch
|
||||
|
||||
@Rule public TemporaryFolder tmp = new TemporaryFolder();
|
||||
|
||||
@ClassRule public static PostgreSQLContainer postgres =
|
||||
new PostgreSQLContainer()
|
||||
@ClassRule
|
||||
public static PostgreSQLContainer postgres =
|
||||
new PostgreSQLContainer(NomulusPostgreSql.getDockerTag())
|
||||
.withDatabaseName("postgres")
|
||||
.withUsername("postgres")
|
||||
.withPassword("domain-registry");
|
||||
|
||||
104
db/README.md
104
db/README.md
@@ -9,16 +9,16 @@ Nomulus uses the 'postgres' database in the 'public' schema. The following
|
||||
users/roles are defined:
|
||||
|
||||
* postgres: the initial user is used for admin and schema deployment.
|
||||
* In Cloud SQL, we do not control superusers. The initial 'postgres' user
|
||||
is a regular user with create-role/create-db privileges. Therefore,
|
||||
it is not possible to separate admin user and schema-deployment user.
|
||||
* In Cloud SQL, we do not control superusers. The initial 'postgres' user
|
||||
is a regular user with create-role/create-db privileges. Therefore, it
|
||||
is not possible to separate admin user and schema-deployment user.
|
||||
* readwrite is a role with read-write privileges on all data tables and
|
||||
sequences. However, it does not have write access to admin tables. Nor
|
||||
can it create new tables.
|
||||
sequences. However, it does not have write access to admin tables. Nor can
|
||||
it create new tables.
|
||||
* The Registry server user is granted this role.
|
||||
* readonly is a role with SELECT privileges on all tables.
|
||||
* Reporting job user and individual human readers may be granted
|
||||
this role.
|
||||
* Reporting job user and individual human readers may be granted this
|
||||
role.
|
||||
|
||||
### Schema DDL Scripts
|
||||
|
||||
@@ -33,23 +33,29 @@ Below are the steps to submit a schema change:
|
||||
`core/src/main/resources/META-INF/persistence.xml` so they'll be picked up.
|
||||
2. Run the `nomulus generate_sql_schema` command to generate a new version of
|
||||
`db-schema.sql.generated`. The full command line to do this is:
|
||||
|
||||
`./gradlew registryTool --args="-e localhost generate_sql_schema --start_postgresql -o /path/to/nomulus/db/src/main/resources/sql/schema/db-schema.sql.generated"`
|
||||
3. Write an incremental DDL script that changes the existing schema to your
|
||||
new one. The generated SQL file from the previous step should help. New
|
||||
create table statements can be used as is, whereas alter table statements
|
||||
should be written to change any existing tables.
|
||||
|
||||
|
||||
`./gradlew registryTool --args="-e localhost generate_sql_schema
|
||||
--start_postgresql -o
|
||||
/path/to/nomulus/db/src/main/resources/sql/schema/db-schema.sql.generated"`
|
||||
|
||||
3. Write an incremental DDL script that changes the existing schema to your new
|
||||
one. The generated SQL file from the previous step should help. New create
|
||||
table statements can be used as is, whereas alter table statements should be
|
||||
written to change any existing tables.
|
||||
|
||||
This script should be stored in a new file in the
|
||||
`db/src/main/resources/sql/flyway` folder using the naming pattern
|
||||
`V{id}__{description text}.sql`, where `{id}` is the next highest number
|
||||
following the existing scripts in that folder. Note the double underscore in
|
||||
the naming pattern.
|
||||
|
||||
4. Run the `:db:test` task from the Gradle root project. The SchemaTest will
|
||||
fail because the new schema does not match the golden file.
|
||||
5. Copy `db/build/resources/test/testcontainer/mount/dump.txt` to the golden file
|
||||
`db/src/main/resources/sql/schema/nomulus.golden.sql`. Diff it against the
|
||||
old version and verify that all changes are expected.
|
||||
|
||||
5. Copy `db/build/resources/test/testcontainer/mount/dump.txt` to the golden
|
||||
file `db/src/main/resources/sql/schema/nomulus.golden.sql`. Diff it against
|
||||
the old version and verify that all changes are expected.
|
||||
|
||||
6. Re-run the `:db:test` task. This time all tests should pass.
|
||||
|
||||
Relevant files (under db/src/main/resources/sql/schema/):
|
||||
@@ -66,23 +72,63 @@ example, when adding a new column to a table, we would deploy the change before
|
||||
adding it to the relevant ORM class. Therefore, for a short time the golden file
|
||||
will contain the new column while the generated one does not.
|
||||
|
||||
### Non-production Schema Push
|
||||
### Schema Push
|
||||
|
||||
To manage schema in a non-production environment, use the 'flywayMigration'
|
||||
task. You will need Cloud SDK and login once.
|
||||
Currently Cloud SQL schema is released with the Nomulus server, and shares the
|
||||
server release's tag (e.g., nomulus-20191101-RC00). Automatic schema push
|
||||
process (to apply new changes in a released schema to the databases) has not
|
||||
been set up yet, and new schema may be pushed manually on demand.
|
||||
|
||||
Presubmit and continuous-integration tests are being implemented to ensure
|
||||
server/schema compatibility. Before the tests are activated, please look for
|
||||
breaking changes before deploying a schema.
|
||||
|
||||
Released schema may be deployed using Cloud Build. Use the root project
|
||||
directory as working directory, run the following shell snippets:
|
||||
|
||||
```shell
|
||||
# Tags exist as folder names under gs://domain-registry-dev-deploy.
|
||||
SCHEMA_TAG=
|
||||
# Recognized environments are alpha, crash, sandbox and production
|
||||
SQL_ENV=
|
||||
# Deploy on cloud build. The --project is optional if domain-registry-dev
|
||||
# is already your default project.
|
||||
gcloud builds submit --config=release/cloudbuild-schema-deploy.yaml \
|
||||
--substitutions=TAG_NAME=${SCHEMA_TAG},_ENV=${SQL_ENV} \
|
||||
--project domain-registry-dev
|
||||
# Verify by checking Flyway Schema History:
|
||||
./gradlew :db:flywayInfo -PdbServer=${SQL_ENV}
|
||||
```
|
||||
|
||||
#### Glass Breaking
|
||||
|
||||
If you need to deploy a schema off-cycle, try making a release first, then
|
||||
deploy that release schema to Cloud SQL.
|
||||
|
||||
TODO(weiminyu): elaborate on different ways to push schema without a full
|
||||
release.
|
||||
|
||||
#### Notes On Flyway
|
||||
|
||||
Please note: to run Flyway commands, you need Cloud SDK and need to log in once.
|
||||
|
||||
```shell
|
||||
# One time login
|
||||
gcloud auth login
|
||||
|
||||
# Deploy the current schema to alpha
|
||||
gradlew :db:flywayMigrate -PdbServer=alpha
|
||||
|
||||
# Delete the entire schema in alpha
|
||||
gradlew :db:flywayClean -PdbServer=alpha
|
||||
```
|
||||
|
||||
The flywayMigrate task is idempotent. Repeated runs will not introduce problems.
|
||||
The Flyway-based Cloud Build schema push process is safe in common scenarios:
|
||||
|
||||
* Repeatedly deploying the latest schema is safe. All duplicate runs become
|
||||
NOP.
|
||||
|
||||
* Accidentally deploying a past schema is safe. Flyway will not undo
|
||||
incremental changes not reflected in the deployed schema.
|
||||
|
||||
* Concurrent deployment runs are safe. Flyway locks its own metadata table,
|
||||
serializing deployment runs without affecting normal accesses.
|
||||
|
||||
#### Schema Push to Local Database
|
||||
|
||||
The Flyway tasks may also be used to deploy to local instances, e.g, your own
|
||||
test instance. E.g.,
|
||||
@@ -95,7 +141,3 @@ gradlew :db:flywayMigrate -PdbServer=192.168.9.2 -PdbPassword=domain-registry
|
||||
gradlew :db:flywayMigrate -PdbServer=192.168.9.2:5432 -PdbUser=postgres \
|
||||
-PdbPassword=domain-registry
|
||||
```
|
||||
|
||||
### Production Schema Deployment
|
||||
|
||||
Schema deployment to production and sandbox is under development.
|
||||
|
||||
@@ -31,24 +31,8 @@ ext {
|
||||
def dbServer = findProperty(dbServerProperty).toString().toLowerCase()
|
||||
def dbName = findProperty(dbNameProperty)
|
||||
|
||||
reconfirmRestrictedDbEnv = {
|
||||
if (!restrictedDbEnv.contains(dbServer)) {
|
||||
return
|
||||
}
|
||||
// For restricted environments, ask the user to type again to confirm.
|
||||
// The following statement uses Gradle internal API to get around the
|
||||
// missing console bug when Gradle Daemon is in use. Another option is
|
||||
// to use the ant.input task. For details please refer to
|
||||
// https://github.com/gradle/gradle/issues/1251.
|
||||
def dbServerAgain = services.get(UserInputHandler.class).askQuestion(
|
||||
"""\
|
||||
Are you sure? Operating on ${dbServer} from desktop is unsafe.
|
||||
Please type '${dbServer}' again to proceed: """.stripIndent(),
|
||||
'').trim()
|
||||
if (dbServer != dbServerAgain) {
|
||||
throw new RuntimeException(
|
||||
"Failed to confirm for restricted database environment. Operation aborted.")
|
||||
}
|
||||
isCloudSql = {
|
||||
return allDbEnv.contains(dbServer)
|
||||
}
|
||||
|
||||
getAccessInfoByHostPort = { hostAndPort ->
|
||||
@@ -87,18 +71,14 @@ ext {
|
||||
// production. The role parameter may be superuser. (More roles will be added
|
||||
// later).
|
||||
getCloudSqlCredential = { env, role ->
|
||||
env = env == 'production' ? '' : "-${env}"
|
||||
def keyProject = env == '-crash'
|
||||
? 'domain-registry-crash-kms-keys'
|
||||
: "domain-registry${env}-keys"
|
||||
def command =
|
||||
"""gsutil cp \
|
||||
gs://domain-registry${env}-cloudsql-credentials/${role}_credential.enc - | \
|
||||
gs://domain-registry-dev-deploy/cloudsql-credentials/${env}/${role}_credential.enc - | \
|
||||
base64 -d | \
|
||||
gcloud kms decrypt --location global --keyring nomulus \
|
||||
--key sql-credentials-on-gcs-key --plaintext-file=- \
|
||||
gcloud kms decrypt --location global --keyring nomulus-tool-keyring \
|
||||
--key nomulus-tool-key --plaintext-file=- \
|
||||
--ciphertext-file=- \
|
||||
--project=${keyProject}"""
|
||||
--project=domain-registry-dev"""
|
||||
|
||||
return execInBash(command, '/tmp')
|
||||
}
|
||||
@@ -112,8 +92,23 @@ task schemaJar(type: Jar) {
|
||||
}
|
||||
}
|
||||
|
||||
// Expose NomulusPostgreSql class to ':core' for compile, without leaking
|
||||
// unnecessary dependencies to the release artifacts through ':core'.
|
||||
// Jar is put in the 'compileApi' configuration.
|
||||
task compileApiJar(type: Jar) {
|
||||
archiveBaseName = 'compile'
|
||||
from(sourceSets.main.output) {
|
||||
include 'google/registry/persistence/NomulusPostgreSql**'
|
||||
}
|
||||
}
|
||||
|
||||
configurations {
|
||||
compileApi
|
||||
}
|
||||
|
||||
artifacts {
|
||||
archives schemaJar
|
||||
compileApi compileApiJar
|
||||
}
|
||||
|
||||
publishing {
|
||||
@@ -132,7 +127,6 @@ publishing {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
flyway {
|
||||
def accessInfo = project.ext.getJdbcAccessInfo()
|
||||
|
||||
@@ -144,13 +138,6 @@ flyway {
|
||||
locations = [ "classpath:sql/flyway" ]
|
||||
}
|
||||
|
||||
tasks.flywayMigrate.dependsOn(
|
||||
tasks.create('confirmMigrateOnRestrictedDb') {
|
||||
doLast {
|
||||
project.ext.reconfirmRestrictedDbEnv()
|
||||
}
|
||||
})
|
||||
|
||||
dependencies {
|
||||
def deps = rootProject.dependencyMap
|
||||
|
||||
@@ -170,7 +157,23 @@ dependencies {
|
||||
testCompile project(':third_party')
|
||||
}
|
||||
|
||||
// Ensure that resources are rebuilt before running Flyway tasks
|
||||
tasks
|
||||
.findAll { task -> task.group.equals('Flyway')}
|
||||
.collect { task -> task.dependsOn('buildNeeded') }
|
||||
flywayValidate.dependsOn('buildNeeded')
|
||||
|
||||
if (ext.isCloudSql()) {
|
||||
// Disable dangerous Flyway tasks. Only allow info and validate.
|
||||
tasks.findAll { task -> task.group.equals('Flyway')}.each {
|
||||
if (it.name == 'flywayMigrate') {
|
||||
it.doFirst {
|
||||
throw new UnsupportedOperationException(
|
||||
""" \
|
||||
FlywayMigrate is disabled. See README.md for schema deployment
|
||||
instructions.""".stripIndent())
|
||||
}
|
||||
} else if (it.name != 'flywayInfo' && it.name != 'flywayValidate') {
|
||||
it.doFirst {
|
||||
throw new UnsupportedOperationException(
|
||||
"${it.name} from commandline is not allowed.")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package google.registry.persistence;
|
||||
|
||||
/** Information about Nomulus' Cloud SQL PostgreSql instance. */
|
||||
public class NomulusPostgreSql {
|
||||
|
||||
/** The current PostgreSql version in Cloud SQL. */
|
||||
// TODO(weiminyu): setup periodic checks to detect version changes in Cloud SQL.
|
||||
private static final String TARGET_VERSION = "11.5";
|
||||
|
||||
/** Returns the docker image tag of the targeted Postgresql server version. */
|
||||
public static String getDockerTag() {
|
||||
return "postgres:" + TARGET_VERSION;
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,8 @@
|
||||
-- PostgreSQL database dump
|
||||
--
|
||||
|
||||
-- Dumped from database version 9.6.12
|
||||
-- Dumped by pg_dump version 9.6.12
|
||||
-- Dumped from database version 11.5 (Debian 11.5-3.pgdg90+1)
|
||||
-- Dumped by pg_dump version 11.5 (Debian 11.5-3.pgdg90+1)
|
||||
|
||||
SET statement_timeout = 0;
|
||||
SET lock_timeout = 0;
|
||||
@@ -12,23 +12,10 @@ SET client_encoding = 'UTF8';
|
||||
SET standard_conforming_strings = on;
|
||||
SELECT pg_catalog.set_config('search_path', '', false);
|
||||
SET check_function_bodies = false;
|
||||
SET xmloption = content;
|
||||
SET client_min_messages = warning;
|
||||
SET row_security = off;
|
||||
|
||||
--
|
||||
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
|
||||
|
||||
|
||||
--
|
||||
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
|
||||
--
|
||||
|
||||
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
|
||||
|
||||
|
||||
SET default_tablespace = '';
|
||||
|
||||
SET default_with_oids = false;
|
||||
|
||||
@@ -19,6 +19,7 @@ import static google.registry.testing.TextDiffSubject.assertThat;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.io.Resources;
|
||||
import google.registry.persistence.NomulusPostgreSql;
|
||||
import java.io.File;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -54,7 +55,7 @@ public class SchemaTest {
|
||||
*/
|
||||
@Rule
|
||||
public PostgreSQLContainer sqlContainer =
|
||||
new PostgreSQLContainer<>("postgres:9.6.12")
|
||||
new PostgreSQLContainer<>(NomulusPostgreSql.getDockerTag())
|
||||
.withClasspathResourceMapping(
|
||||
MOUNTED_RESOURCE_PATH, CONTAINER_MOUNT_POINT, BindMode.READ_WRITE);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user