Add smoke test for BEAM pipelines (#3037)

Created a smoke test to cover unit test gaps wrt BEAM:
- The Java and SDK compatibility in the pipeline container image
- The JPA setup in the pipelines

Both issues above can only be tested in a real pipeline.

This PR defines a new pipeline that performs a lightweight SQL
query and minimal processing. The build process can launch it
in a test environment to verify that the pipelines in the build
can run. The run script is also provided.
This commit is contained in:
Weimin Yu
2026-05-07 20:41:13 +00:00
committed by GitHub
parent 80eefc6498
commit 60d3653b46
7 changed files with 284 additions and 3 deletions
@@ -0,0 +1,76 @@
// Copyright 2026 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.beam.common;
import static com.google.common.base.Verify.verify;
import com.google.common.flogger.FluentLogger;
import google.registry.model.tld.Tld;
import google.registry.persistence.transaction.CriteriaQueryBuilder;
import java.io.Serializable;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
/**
* For smoke test in the build/deployment process.
*
* <p>There two coverage gaps in unit tests for BEAM pipelines:
*
* <ul>
* <li>The compatibility of the JVM and SDK in the pipeline image
* <li>The JPA setup, which is performed by the {@link RegistryPipelineWorkerInitializer}
* </ul>
*
* <p>This classes defines a pipeline that performs one quick database query. The pipeline is
* expected to complete quickly, and the build or deployment process may launch it on GCP and wait
* for its completion to be certain that all aspects are tested for Nomulus pipelines.
*/
public class SmokeTestPipeline implements Serializable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
public static void main(String[] args) {
PipelineOptionsFactory.register(RegistryPipelineOptions.class);
RegistryPipelineOptions options =
PipelineOptionsFactory.fromArgs(args).withValidation().as(RegistryPipelineOptions.class);
runPipeline(options);
}
static PipelineResult runPipeline(RegistryPipelineOptions options) {
Pipeline pipeline = Pipeline.create(options);
pipeline
.apply(
"Read Tlds",
RegistryJpaIO.read(() -> CriteriaQueryBuilder.create(Tld.class).build(), Tld::getTldStr)
.withCoder(StringUtf8Coder.of()))
.apply("Count Tlds", Count.globally())
.apply(
"Verify Count",
ParDo.of(
new DoFn<Long, Void>() {
@DoFn.ProcessElement
public void processElement(@Element Long count) {
logger.atInfo().log("Tld count: %s", count);
verify(count > 0, "Expecting 1 or more, got %s.", count);
}
}));
return pipeline.run();
}
}
@@ -0,0 +1,24 @@
{
"name": "Beam pipeline smoke test",
"description": "An Apache Beam pipeline that performs a simple database query.",
"parameters": [
{
"name": "registryEnvironment",
"label": "The Registry environment.",
"helpText": "The Registry environment.",
"is_optional": false,
"regexes": [
"^SANDBOX|CRASH$"
]
},
{
"name": "isolationOverride",
"label": "The desired SQL transaction isolation level.",
"helpText": "The desired SQL transaction isolation level.",
"is_optional": true,
"regexes": [
"^[0-9A-Z_]+$"
]
}
]
}
@@ -0,0 +1,64 @@
// Copyright 2026 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.beam.common;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ;
import static google.registry.testing.DatabaseHelper.createTld;
import static org.junit.jupiter.api.Assertions.assertThrows;
import google.registry.beam.TestPipelineExtension;
import google.registry.persistence.transaction.JpaTestExtensions;
import google.registry.testing.FakeClock;
import java.time.Instant;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.hibernate.cfg.AvailableSettings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
public class SmokeTestPipelineTest {
private final FakeClock clock = new FakeClock(Instant.parse("2021-02-02T00:00:05.000Z"));
@RegisterExtension
final JpaTestExtensions.JpaIntegrationTestExtension jpa =
new JpaTestExtensions.Builder()
.withClock(clock)
.withProperty(AvailableSettings.ISOLATION, TRANSACTION_REPEATABLE_READ.name())
.buildIntegrationTestExtension();
@RegisterExtension
final TestPipelineExtension pipeline =
TestPipelineExtension.create().enableAbandonedNodeEnforcement(true);
private final RegistryPipelineOptions options =
PipelineOptionsFactory.create().as(RegistryPipelineOptions.class);
@Test
void whenIldsDoNotExist_failure() {
var exception =
assertThrows(
Pipeline.PipelineExecutionException.class,
() -> SmokeTestPipeline.runPipeline(options).waitUntilFinish());
assertThat(exception).hasMessageThat().contains("Expecting 1 or more, got 0.");
}
@Test
void whenTldsExist_success() {
createTld("tld");
SmokeTestPipeline.runPipeline(options).waitUntilFinish();
}
}