diff --git a/java/com/google/domain/registry/bigquery/BigqueryFactory.java b/java/com/google/domain/registry/bigquery/BigqueryFactory.java index 803139389..cef9bc93e 100644 --- a/java/com/google/domain/registry/bigquery/BigqueryFactory.java +++ b/java/com/google/domain/registry/bigquery/BigqueryFactory.java @@ -16,7 +16,8 @@ package com.google.domain.registry.bigquery; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.domain.registry.bigquery.BigquerySchemas.knownTableSchemas; +import static com.google.common.collect.Sets.newConcurrentHashSet; +import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass; import com.google.api.client.extensions.appengine.http.UrlFetchTransport; import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; @@ -32,13 +33,12 @@ import com.google.api.services.bigquery.model.Table; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableReference; import com.google.api.services.bigquery.model.TableSchema; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Sets; import com.google.domain.registry.util.FormattingLogger; -import com.google.domain.registry.util.NonFinalForTesting; import java.io.IOException; +import java.util.List; +import java.util.Map; import java.util.Set; import javax.inject.Inject; @@ -46,24 +46,21 @@ import javax.inject.Inject; /** Factory for creating {@link Bigquery} connections. */ public class BigqueryFactory { - private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + private static final FormattingLogger logger = getLoggerForCallerClass(); // Cross-request caches to avoid unnecessary RPCs. - @NonFinalForTesting - private Set knownTables = Sets.newConcurrentHashSet(); - - @NonFinalForTesting - private Set datasets = Sets.newConcurrentHashSet(); - - @NonFinalForTesting - @VisibleForTesting - Subfactory subfactory = new Subfactory(); + private static Set knownExistingDatasets = newConcurrentHashSet(); + private static Set knownExistingTables = newConcurrentHashSet(); + @Inject Map> bigquerySchemas; + @Inject Subfactory subfactory; @Inject BigqueryFactory() {} /** This class is broken out solely so that it can be mocked inside of tests. */ static class Subfactory { + @Inject Subfactory() {} + public Bigquery create( String applicationName, HttpTransport transport, @@ -96,9 +93,9 @@ public class BigqueryFactory { new AppIdentityCredential(BigqueryScopes.all())); // Note: it's safe for multiple threads to call this as the dataset will only be created once. - if (!datasets.contains(datasetId)) { + if (!knownExistingDatasets.contains(datasetId)) { ensureDataset(bigquery, projectId, datasetId); - datasets.add(datasetId); + knownExistingDatasets.add(datasetId); } return bigquery; @@ -111,17 +108,17 @@ public class BigqueryFactory { public Bigquery create(String projectId, String datasetId, String tableId) throws IOException { Bigquery bigquery = create(projectId, datasetId); - checkArgument(knownTableSchemas.containsKey(tableId), "Unknown table ID: %s", tableId); + checkArgument(bigquerySchemas.containsKey(tableId), "Unknown table ID: %s", tableId); - if (!knownTables.contains(tableId)) { + if (!knownExistingTables.contains(tableId)) { ensureTable( bigquery, new TableReference() .setDatasetId(datasetId) .setProjectId(projectId) .setTableId(tableId), - knownTableSchemas.get(tableId)); - knownTables.add(tableId); + bigquerySchemas.get(tableId)); + knownExistingTables.add(tableId); } return bigquery; @@ -151,8 +148,7 @@ public class BigqueryFactory { } /** Ensures the table exists in Bigquery. */ - private void ensureTable( - Bigquery bigquery, TableReference table, ImmutableList schema) + private void ensureTable(Bigquery bigquery, TableReference table, List schema) throws IOException { try { bigquery.tables().insert(table.getProjectId(), table.getDatasetId(), new Table() diff --git a/java/com/google/domain/registry/bigquery/BigqueryModule.java b/java/com/google/domain/registry/bigquery/BigqueryModule.java index fcb4ac711..aa33e3e98 100644 --- a/java/com/google/domain/registry/bigquery/BigqueryModule.java +++ b/java/com/google/domain/registry/bigquery/BigqueryModule.java @@ -21,12 +21,16 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.services.bigquery.Bigquery; import com.google.api.services.bigquery.BigqueryScopes; +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.common.collect.ImmutableList; import com.google.domain.registry.config.ConfigModule.Config; import com.google.domain.registry.request.OAuthScopes; import dagger.Module; +import dagger.Multibindings; import dagger.Provides; +import java.util.Map; import java.util.Set; /** @@ -41,6 +45,13 @@ import java.util.Set; @Module public final class BigqueryModule { + @Multibindings + interface BigQueryMultibindings { + + /** Provides a map of BigQuery table names to field names. */ + Map> bigquerySchemas(); + } + /** Provides OAuth2 scopes for the Bigquery service needed by Domain Registry. */ @Provides(type = SET_VALUES) @OAuthScopes diff --git a/java/com/google/domain/registry/bigquery/BigquerySchemas.java b/java/com/google/domain/registry/bigquery/BigquerySchemas.java deleted file mode 100644 index 2c5ef5d2e..000000000 --- a/java/com/google/domain/registry/bigquery/BigquerySchemas.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2016 The Domain Registry Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.domain.registry.bigquery; - -import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.domain.registry.bigquery.BigqueryUtils.FieldType; -import com.google.domain.registry.util.NonFinalForTesting; - -import java.util.Map; - -/** Schemas for BigQuery tables. */ -public final class BigquerySchemas { - - public static final String EPPMETRICS_TABLE_ID = "eppMetrics"; - public static final String ENTITY_INTEGRITY_ALERTS_TABLE_ID = "alerts"; - public static final String ENTITY_INTEGRITY_ALERTS_FIELD_SCANTIME = "scanTime"; - public static final String ENTITY_INTEGRITY_ALERTS_FIELD_SOURCE = "source"; - public static final String ENTITY_INTEGRITY_ALERTS_FIELD_TARGET = "target"; - public static final String ENTITY_INTEGRITY_ALERTS_FIELD_MESSAGE = "message"; - - static final ImmutableList EPPMETRICS_SCHEMA_FIELDS = - ImmutableList.of( - new TableFieldSchema().setName("requestId").setType(FieldType.STRING.name()), - new TableFieldSchema().setName("startTime").setType(FieldType.TIMESTAMP.name()), - new TableFieldSchema().setName("endTime").setType(FieldType.TIMESTAMP.name()), - new TableFieldSchema().setName("commandName").setType(FieldType.STRING.name()), - new TableFieldSchema().setName("clientId").setType(FieldType.STRING.name()), - new TableFieldSchema().setName("privilegeLevel").setType(FieldType.STRING.name()), - new TableFieldSchema().setName("eppTarget").setType(FieldType.STRING.name()), - new TableFieldSchema().setName("eppStatus").setType(FieldType.INTEGER.name()), - new TableFieldSchema().setName("attempts").setType(FieldType.INTEGER.name())); - - static final ImmutableList ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS = - ImmutableList.of( - new TableFieldSchema() - .setName(ENTITY_INTEGRITY_ALERTS_FIELD_SCANTIME) - .setType(FieldType.TIMESTAMP.name()), - new TableFieldSchema() - .setName(ENTITY_INTEGRITY_ALERTS_FIELD_SOURCE) - .setType(FieldType.STRING.name()), - new TableFieldSchema() - .setName(ENTITY_INTEGRITY_ALERTS_FIELD_TARGET) - .setType(FieldType.STRING.name()), - new TableFieldSchema() - .setName(ENTITY_INTEGRITY_ALERTS_FIELD_MESSAGE) - .setType(FieldType.STRING.name())); - - @NonFinalForTesting - static Map> knownTableSchemas = - new ImmutableMap.Builder>() - .put(EPPMETRICS_TABLE_ID, EPPMETRICS_SCHEMA_FIELDS) - .put(ENTITY_INTEGRITY_ALERTS_TABLE_ID, ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS) - .build(); - - private BigquerySchemas() {} -} diff --git a/java/com/google/domain/registry/monitoring/whitebox/EntityIntegrityAlertsSchema.java b/java/com/google/domain/registry/monitoring/whitebox/EntityIntegrityAlertsSchema.java new file mode 100644 index 000000000..07b1edfb9 --- /dev/null +++ b/java/com/google/domain/registry/monitoring/whitebox/EntityIntegrityAlertsSchema.java @@ -0,0 +1,42 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.domain.registry.monitoring.whitebox; + +import static com.google.domain.registry.bigquery.BigqueryUtils.FieldType.STRING; +import static com.google.domain.registry.bigquery.BigqueryUtils.FieldType.TIMESTAMP; + +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.common.collect.ImmutableList; + +/** The Bigquery schema for the entity integrity alerts table. */ +final class EntityIntegrityAlertsSchema { + + static final String DATASET = "entity_integrity"; + static final String TABLE_ID = "alerts"; + static final String FIELD_SCANTIME = "scanTime"; + static final String FIELD_SOURCE = "source"; + static final String FIELD_TARGET = "target"; + static final String FIELD_MESSAGE = "message"; + + static final ImmutableList ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS = + ImmutableList.of( + new TableFieldSchema().setName(FIELD_SCANTIME).setType(TIMESTAMP.name()), + new TableFieldSchema().setName(FIELD_SOURCE).setType(STRING.name()), + new TableFieldSchema().setName(FIELD_TARGET).setType(STRING.name()), + new TableFieldSchema().setName(FIELD_MESSAGE).setType(STRING.name())); + + private EntityIntegrityAlertsSchema() {} +} + diff --git a/java/com/google/domain/registry/monitoring/whitebox/EppMetrics.java b/java/com/google/domain/registry/monitoring/whitebox/EppMetrics.java index dc02885e3..d6d63395c 100644 --- a/java/com/google/domain/registry/monitoring/whitebox/EppMetrics.java +++ b/java/com/google/domain/registry/monitoring/whitebox/EppMetrics.java @@ -14,14 +14,30 @@ package com.google.domain.registry.monitoring.whitebox; -import com.google.domain.registry.bigquery.BigquerySchemas; +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.common.collect.ImmutableList; +import com.google.domain.registry.bigquery.BigqueryUtils.FieldType; import com.google.domain.registry.model.eppoutput.Result.Code; /** The EPP Metrics collector. See {@link Metrics}. */ public class EppMetrics extends Metrics { + static final String EPPMETRICS_TABLE_ID = "eppMetrics"; + + static final ImmutableList EPPMETRICS_SCHEMA_FIELDS = + ImmutableList.of( + new TableFieldSchema().setName("requestId").setType(FieldType.STRING.name()), + new TableFieldSchema().setName("startTime").setType(FieldType.TIMESTAMP.name()), + new TableFieldSchema().setName("endTime").setType(FieldType.TIMESTAMP.name()), + new TableFieldSchema().setName("commandName").setType(FieldType.STRING.name()), + new TableFieldSchema().setName("clientId").setType(FieldType.STRING.name()), + new TableFieldSchema().setName("privilegeLevel").setType(FieldType.STRING.name()), + new TableFieldSchema().setName("eppTarget").setType(FieldType.STRING.name()), + new TableFieldSchema().setName("eppStatus").setType(FieldType.INTEGER.name()), + new TableFieldSchema().setName("attempts").setType(FieldType.INTEGER.name())); + public EppMetrics() { - setTableId(BigquerySchemas.EPPMETRICS_TABLE_ID); + setTableId(EPPMETRICS_TABLE_ID); fields.put("attempts", 0); } diff --git a/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityStreamer.java b/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityStreamer.java index dcdb73bfa..8af1d1281 100644 --- a/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityStreamer.java +++ b/java/com/google/domain/registry/monitoring/whitebox/VerifyEntityIntegrityStreamer.java @@ -15,11 +15,12 @@ package com.google.domain.registry.monitoring.whitebox; import static com.google.api.client.util.Data.NULL_STRING; -import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_FIELD_MESSAGE; -import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_FIELD_SCANTIME; -import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_FIELD_SOURCE; -import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_FIELD_TARGET; -import static com.google.domain.registry.bigquery.BigquerySchemas.ENTITY_INTEGRITY_ALERTS_TABLE_ID; +import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.DATASET; +import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.FIELD_MESSAGE; +import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.FIELD_SCANTIME; +import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.FIELD_SOURCE; +import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.FIELD_TARGET; +import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.TABLE_ID; import com.google.api.services.bigquery.Bigquery; import com.google.api.services.bigquery.Bigquery.Tabledata.InsertAll; @@ -54,8 +55,6 @@ import javax.annotation.Nullable; @AutoFactory(allowSubclasses = true) public class VerifyEntityIntegrityStreamer { - private static final String DATASET = "entity_integrity"; - private final DateTime scanTime; private Bigquery bigquery; @@ -84,7 +83,7 @@ public class VerifyEntityIntegrityStreamer { if (bigquery == null) { bigquery = bigqueryFactory.create( - environment.config().getProjectId(), DATASET, ENTITY_INTEGRITY_ALERTS_TABLE_ID); + environment.config().getProjectId(), DATASET, TABLE_ID); } return bigquery; } @@ -157,16 +156,16 @@ public class VerifyEntityIntegrityStreamer { Map rowData = new ImmutableMap.Builder() .put( - ENTITY_INTEGRITY_ALERTS_FIELD_SCANTIME, + FIELD_SCANTIME, new com.google.api.client.util.DateTime(scanTime.toDate())) .put( - ENTITY_INTEGRITY_ALERTS_FIELD_SOURCE, + FIELD_SOURCE, source.toString()) .put( - ENTITY_INTEGRITY_ALERTS_FIELD_TARGET, + FIELD_TARGET, target.toString()) .put( - ENTITY_INTEGRITY_ALERTS_FIELD_MESSAGE, + FIELD_MESSAGE, (message == null) ? NULL_STRING : message) .build(); rows.add( @@ -185,7 +184,7 @@ public class VerifyEntityIntegrityStreamer { .insertAll( environment.config().getProjectId(), DATASET, - ENTITY_INTEGRITY_ALERTS_TABLE_ID, + TABLE_ID, new TableDataInsertAllRequest().setRows(rows)); Callable callable = diff --git a/java/com/google/domain/registry/monitoring/whitebox/WhiteboxModule.java b/java/com/google/domain/registry/monitoring/whitebox/WhiteboxModule.java index 3f4ce059a..421954292 100644 --- a/java/com/google/domain/registry/monitoring/whitebox/WhiteboxModule.java +++ b/java/com/google/domain/registry/monitoring/whitebox/WhiteboxModule.java @@ -14,13 +14,21 @@ package com.google.domain.registry.monitoring.whitebox; +import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS; +import static com.google.domain.registry.monitoring.whitebox.EntityIntegrityAlertsSchema.TABLE_ID; +import static com.google.domain.registry.monitoring.whitebox.EppMetrics.EPPMETRICS_SCHEMA_FIELDS; +import static com.google.domain.registry.monitoring.whitebox.EppMetrics.EPPMETRICS_TABLE_ID; import static com.google.domain.registry.request.RequestParameters.extractRequiredParameter; +import static dagger.Provides.Type.MAP; +import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; import com.google.domain.registry.request.Parameter; import dagger.Module; import dagger.Provides; +import dagger.multibindings.StringKey; import java.util.UUID; @@ -32,6 +40,18 @@ import javax.servlet.http.HttpServletRequest; @Module public class WhiteboxModule { + @Provides(type = MAP) + @StringKey(EPPMETRICS_TABLE_ID) + static ImmutableList provideEppMetricsSchema() { + return EPPMETRICS_SCHEMA_FIELDS; + } + + @Provides(type = MAP) + @StringKey(TABLE_ID) + static ImmutableList provideEntityIntegrityAlertsSchema() { + return ENTITY_INTEGRITY_ALERTS_SCHEMA_FIELDS; + } + @Provides @Parameter("tableId") static String provideTableId(HttpServletRequest req) { diff --git a/javatests/com/google/domain/registry/bigquery/BigqueryFactoryTest.java b/javatests/com/google/domain/registry/bigquery/BigqueryFactoryTest.java index 271344bec..4c92b7d86 100644 --- a/javatests/com/google/domain/registry/bigquery/BigqueryFactoryTest.java +++ b/javatests/com/google/domain/registry/bigquery/BigqueryFactoryTest.java @@ -32,10 +32,8 @@ import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableReference; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.domain.registry.testing.InjectRule; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; @@ -46,9 +44,6 @@ import org.mockito.runners.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public class BigqueryFactoryTest { - @Rule - public final InjectRule inject = new InjectRule(); - @Mock private BigqueryFactory.Subfactory subfactory; @@ -67,6 +62,8 @@ public class BigqueryFactoryTest { @Mock private Bigquery.Tables.Insert bigqueryTablesInsert; + private BigqueryFactory factory; + @Before public void before() throws Exception { when(subfactory.create( @@ -81,16 +78,21 @@ public class BigqueryFactoryTest { when(bigquery.tables()).thenReturn(bigqueryTables); when(bigqueryTables.insert(eq("Project-Id"), any(String.class), any(Table.class))) .thenReturn(bigqueryTablesInsert); - BigquerySchemas.knownTableSchemas = - ImmutableMap.of( - "Table-Id", - ImmutableList.of(new TableFieldSchema().setName("column1").setType(STRING.name()))); + factory = new BigqueryFactory(); + factory.subfactory = subfactory; + factory.bigquerySchemas = + new ImmutableMap.Builder>() + .put( + "Table-Id", + ImmutableList.of(new TableFieldSchema().setName("column1").setType(STRING.name()))) + .put( + "Table2", + ImmutableList.of(new TableFieldSchema().setName("column1").setType(STRING.name()))) + .build(); } @Test public void testSuccess_datastoreCreation() throws Exception { - BigqueryFactory factory = new BigqueryFactory(); - factory.subfactory = subfactory; factory.create("Project-Id", "Dataset-Id"); ArgumentCaptor datasetArg = ArgumentCaptor.forClass(Dataset.class); @@ -104,24 +106,22 @@ public class BigqueryFactoryTest { @Test public void testSuccess_datastoreAndTableCreation() throws Exception { - BigqueryFactory factory = new BigqueryFactory(); - factory.subfactory = subfactory; - factory.create("Project-Id", "Dataset-Id", "Table-Id"); + factory.create("Project-Id", "Dataset2", "Table2"); ArgumentCaptor datasetArg = ArgumentCaptor.forClass(Dataset.class); verify(bigqueryDatasets).insert(eq("Project-Id"), datasetArg.capture()); assertThat(datasetArg.getValue().getDatasetReference().getProjectId()) .isEqualTo("Project-Id"); assertThat(datasetArg.getValue().getDatasetReference().getDatasetId()) - .isEqualTo("Dataset-Id"); + .isEqualTo("Dataset2"); verify(bigqueryDatasetsInsert).execute(); ArgumentCaptor tableArg = ArgumentCaptor.forClass(Table.class); - verify(bigqueryTables).insert(eq("Project-Id"), eq("Dataset-Id"), tableArg.capture()); + verify(bigqueryTables).insert(eq("Project-Id"), eq("Dataset2"), tableArg.capture()); TableReference ref = tableArg.getValue().getTableReference(); assertThat(ref.getProjectId()).isEqualTo("Project-Id"); - assertThat(ref.getDatasetId()).isEqualTo("Dataset-Id"); - assertThat(ref.getTableId()).isEqualTo("Table-Id"); + assertThat(ref.getDatasetId()).isEqualTo("Dataset2"); + assertThat(ref.getTableId()).isEqualTo("Table2"); assertThat(tableArg.getValue().getSchema().getFields()) .containsExactly(new TableFieldSchema().setName("column1").setType(STRING.name())); verify(bigqueryTablesInsert).execute(); diff --git a/javatests/com/google/domain/registry/export/UpdateSnapshotViewActionTest.java b/javatests/com/google/domain/registry/export/UpdateSnapshotViewActionTest.java index 457818e91..2dec98123 100644 --- a/javatests/com/google/domain/registry/export/UpdateSnapshotViewActionTest.java +++ b/javatests/com/google/domain/registry/export/UpdateSnapshotViewActionTest.java @@ -113,6 +113,10 @@ public class UpdateSnapshotViewActionTest { public void testSuccess_doPost() throws Exception { action.run(); + // Check that the BigQuery factory was called in such a way that the dataset would be created + // if it didn't already exist. + verify(bigqueryFactory).create("Project-Id", "latest_snapshot"); + // Check that we updated the view. ArgumentCaptor
tableArg = ArgumentCaptor.forClass(Table.class); verify(bigqueryTables).update(