mirror of
https://github.com/google/nomulus
synced 2026-09-20 15:04:24 +00:00
Use shared jar to stage BEAM pipeline if possible (#1008)
* Use shared jar to stage BEAM pipeline if possible Allow multiple BEAM pipelines with the same classes and dependencies to share one Uber jar. Added metadata for BulkDeleteDatastorePipeline. Updated shell and Cloud Build scripts to stage all pipelines in one step.
This commit is contained in:
+3
-3
@@ -78,7 +78,7 @@ import org.apache.beam.sdk.values.TupleTagList;
|
||||
* types in the Datastore using the {@code --numOfKindsHint} argument. If the default value for this
|
||||
* parameter is too low, performance will suffer.
|
||||
*/
|
||||
public class BulkDeletePipeline {
|
||||
public class BulkDeleteDatastorePipeline {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
// This tool is not for use in our critical projects.
|
||||
@@ -89,7 +89,7 @@ public class BulkDeletePipeline {
|
||||
|
||||
private final Pipeline pipeline;
|
||||
|
||||
BulkDeletePipeline(BulkDeletePipelineOptions options) {
|
||||
BulkDeleteDatastorePipeline(BulkDeletePipelineOptions options) {
|
||||
this.options = options;
|
||||
pipeline = Pipeline.create(options);
|
||||
}
|
||||
@@ -303,7 +303,7 @@ public class BulkDeletePipeline {
|
||||
public static void main(String[] args) {
|
||||
BulkDeletePipelineOptions options =
|
||||
PipelineOptionsFactory.fromArgs(args).withValidation().as(BulkDeletePipelineOptions.class);
|
||||
BulkDeletePipeline pipeline = new BulkDeletePipeline(options);
|
||||
BulkDeleteDatastorePipeline pipeline = new BulkDeleteDatastorePipeline(options);
|
||||
pipeline.run();
|
||||
System.exit(0);
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "Bulk Delete Cloud Datastore",
|
||||
"description": "An Apache Beam batch pipeline that deletes Cloud Datastore in bulk. This is easier to use than the GCP-provided template.",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "kindsToDelete",
|
||||
"label": "The data KINDs to delete.",
|
||||
"helpText": "The Datastore KINDs to be deleted. The format may be: the list of kinds to be deleted as a comma-separated string; or '*', which causes all kinds to be deleted."
|
||||
},
|
||||
{
|
||||
"name": "getNumOfKindsHint",
|
||||
"label": "An estimate of the number of KINDs to be deleted.",
|
||||
"helpText": "An estimate of the number of KINDs to be deleted. This is recommended if --kindsToDelete is '*' and the default value is too low.",
|
||||
"is_optional": true,
|
||||
"regexes": [
|
||||
"^[1-9][0-9]*$"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user