mirror of
https://github.com/google/nomulus
synced 2026-09-19 14:34:18 +00:00
Add smoke test for BEAM pipelines (#3037)
Created a smoke test to cover unit test gaps wrt BEAM: - The Java and SDK compatibility in the pipeline container image - The JPA setup in the pipelines Both issues above can only be tested in a real pipeline. This PR defines a new pipeline that performs a lightweight SQL query and minimal processing. The build process can launch it in a test environment to verify that the pipelines in the build can run. The run script is also provided.
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
// Copyright 2026 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.common;
|
||||
|
||||
import static com.google.common.base.Verify.verify;
|
||||
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.model.tld.Tld;
|
||||
import google.registry.persistence.transaction.CriteriaQueryBuilder;
|
||||
import java.io.Serializable;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.PipelineResult;
|
||||
import org.apache.beam.sdk.coders.StringUtf8Coder;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.transforms.Count;
|
||||
import org.apache.beam.sdk.transforms.DoFn;
|
||||
import org.apache.beam.sdk.transforms.ParDo;
|
||||
|
||||
/**
|
||||
* For smoke test in the build/deployment process.
|
||||
*
|
||||
* <p>There two coverage gaps in unit tests for BEAM pipelines:
|
||||
*
|
||||
* <ul>
|
||||
* <li>The compatibility of the JVM and SDK in the pipeline image
|
||||
* <li>The JPA setup, which is performed by the {@link RegistryPipelineWorkerInitializer}
|
||||
* </ul>
|
||||
*
|
||||
* <p>This classes defines a pipeline that performs one quick database query. The pipeline is
|
||||
* expected to complete quickly, and the build or deployment process may launch it on GCP and wait
|
||||
* for its completion to be certain that all aspects are tested for Nomulus pipelines.
|
||||
*/
|
||||
public class SmokeTestPipeline implements Serializable {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
public static void main(String[] args) {
|
||||
PipelineOptionsFactory.register(RegistryPipelineOptions.class);
|
||||
RegistryPipelineOptions options =
|
||||
PipelineOptionsFactory.fromArgs(args).withValidation().as(RegistryPipelineOptions.class);
|
||||
runPipeline(options);
|
||||
}
|
||||
|
||||
static PipelineResult runPipeline(RegistryPipelineOptions options) {
|
||||
Pipeline pipeline = Pipeline.create(options);
|
||||
pipeline
|
||||
.apply(
|
||||
"Read Tlds",
|
||||
RegistryJpaIO.read(() -> CriteriaQueryBuilder.create(Tld.class).build(), Tld::getTldStr)
|
||||
.withCoder(StringUtf8Coder.of()))
|
||||
.apply("Count Tlds", Count.globally())
|
||||
.apply(
|
||||
"Verify Count",
|
||||
ParDo.of(
|
||||
new DoFn<Long, Void>() {
|
||||
@DoFn.ProcessElement
|
||||
public void processElement(@Element Long count) {
|
||||
logger.atInfo().log("Tld count: %s", count);
|
||||
verify(count > 0, "Expecting 1 or more, got %s.", count);
|
||||
}
|
||||
}));
|
||||
|
||||
return pipeline.run();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "Beam pipeline smoke test",
|
||||
"description": "An Apache Beam pipeline that performs a simple database query.",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "registryEnvironment",
|
||||
"label": "The Registry environment.",
|
||||
"helpText": "The Registry environment.",
|
||||
"is_optional": false,
|
||||
"regexes": [
|
||||
"^SANDBOX|CRASH$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "isolationOverride",
|
||||
"label": "The desired SQL transaction isolation level.",
|
||||
"helpText": "The desired SQL transaction isolation level.",
|
||||
"is_optional": true,
|
||||
"regexes": [
|
||||
"^[0-9A-Z_]+$"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright 2026 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.common;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import google.registry.beam.TestPipelineExtension;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions;
|
||||
import google.registry.testing.FakeClock;
|
||||
import java.time.Instant;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.hibernate.cfg.AvailableSettings;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
public class SmokeTestPipelineTest {
|
||||
|
||||
private final FakeClock clock = new FakeClock(Instant.parse("2021-02-02T00:00:05.000Z"));
|
||||
|
||||
@RegisterExtension
|
||||
final JpaTestExtensions.JpaIntegrationTestExtension jpa =
|
||||
new JpaTestExtensions.Builder()
|
||||
.withClock(clock)
|
||||
.withProperty(AvailableSettings.ISOLATION, TRANSACTION_REPEATABLE_READ.name())
|
||||
.buildIntegrationTestExtension();
|
||||
|
||||
@RegisterExtension
|
||||
final TestPipelineExtension pipeline =
|
||||
TestPipelineExtension.create().enableAbandonedNodeEnforcement(true);
|
||||
|
||||
private final RegistryPipelineOptions options =
|
||||
PipelineOptionsFactory.create().as(RegistryPipelineOptions.class);
|
||||
|
||||
@Test
|
||||
void whenIldsDoNotExist_failure() {
|
||||
var exception =
|
||||
assertThrows(
|
||||
Pipeline.PipelineExecutionException.class,
|
||||
() -> SmokeTestPipeline.runPipeline(options).waitUntilFinish());
|
||||
assertThat(exception).hasMessageThat().contains("Expecting 1 or more, got 0.");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenTldsExist_success() {
|
||||
createTld("tld");
|
||||
SmokeTestPipeline.runPipeline(options).waitUntilFinish();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user