From d8c1501213da204b92fc3df239038c0af076d2a1 Mon Sep 17 00:00:00 2001 From: dxy Date: Thu, 31 Aug 2017 14:11:29 -0700 Subject: [PATCH] Add PollMapreduceAction This is the first in a series of CLs containing code from an old CL of Dai's that had never been completed, which compares zone data between Datastore and DNS. I had written a script to do this by calling two nomulus commands, but maybe it can be done directly in Java, which would be convenient. This CL is just the plumbing to check on the status of a Mapreduce. We will need this to know that we can proceed with the next step of comparing the output to the DNS data. Cloned from CL 134295050 by 'g4 patch'. Original change by dxy@dxy:zoneman-reader:1939:citc on 2016/09/26 10:34:22. Add a command for comparing zone data between DNS and datastore ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=167188979 --- .../registry/env/common/tools/WEB-INF/web.xml | 5 + .../module/tools/ToolsRequestComponent.java | 2 + java/google/registry/tools/server/BUILD | 1 + .../tools/server/PollMapreduceAction.java | 58 ++++++++++++ .../tools/server/ToolsServerModule.java | 6 ++ .../module/tools/testdata/tools_routing.txt | 1 + javatests/google/registry/tools/server/BUILD | 1 + .../tools/server/PollMapreduceActionTest.java | 91 +++++++++++++++++++ 8 files changed, 165 insertions(+) create mode 100644 java/google/registry/tools/server/PollMapreduceAction.java create mode 100644 javatests/google/registry/tools/server/PollMapreduceActionTest.java diff --git a/java/google/registry/env/common/tools/WEB-INF/web.xml b/java/google/registry/env/common/tools/WEB-INF/web.xml index 32a6f2ca8..1bc691d46 100644 --- a/java/google/registry/env/common/tools/WEB-INF/web.xml +++ b/java/google/registry/env/common/tools/WEB-INF/web.xml @@ -128,6 +128,11 @@ + + tools-servlet + /_dr/task/pollMapreduce + + diff --git a/java/google/registry/module/tools/ToolsRequestComponent.java b/java/google/registry/module/tools/ToolsRequestComponent.java index 189dba605..d345e9d67 100644 --- a/java/google/registry/module/tools/ToolsRequestComponent.java +++ b/java/google/registry/module/tools/ToolsRequestComponent.java @@ -42,6 +42,7 @@ import google.registry.tools.server.ListPremiumListsAction; import google.registry.tools.server.ListRegistrarsAction; import google.registry.tools.server.ListReservedListsAction; import google.registry.tools.server.ListTldsAction; +import google.registry.tools.server.PollMapreduceAction; import google.registry.tools.server.RefreshDnsForAllDomainsAction; import google.registry.tools.server.ResaveAllEppResourcesAction; import google.registry.tools.server.ToolsServerModule; @@ -77,6 +78,7 @@ interface ToolsRequestComponent { ListReservedListsAction listReservedListsAction(); ListTldsAction listTldsAction(); LoadTestAction loadTestAction(); + PollMapreduceAction pollMapReduceAction(); PublishDetailReportAction publishDetailReportAction(); RefreshDnsForAllDomainsAction refreshDnsForAllDomainsAction(); ResaveAllEppResourcesAction resaveAllEppResourcesAction(); diff --git a/java/google/registry/tools/server/BUILD b/java/google/registry/tools/server/BUILD index 9630bf4a1..451797a8d 100644 --- a/java/google/registry/tools/server/BUILD +++ b/java/google/registry/tools/server/BUILD @@ -25,6 +25,7 @@ java_library( "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_tools_appengine_gcs_client", "@com_google_appengine_tools_appengine_mapreduce", + "@com_google_appengine_tools_appengine_pipeline", "@com_google_code_findbugs_jsr305", "@com_google_dagger", "@com_google_guava", diff --git a/java/google/registry/tools/server/PollMapreduceAction.java b/java/google/registry/tools/server/PollMapreduceAction.java new file mode 100644 index 000000000..9cc3d60fe --- /dev/null +++ b/java/google/registry/tools/server/PollMapreduceAction.java @@ -0,0 +1,58 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.server; + +import static com.google.appengine.tools.pipeline.PipelineServiceFactory.newPipelineService; +import static google.registry.request.Action.Method.POST; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.appengine.tools.mapreduce.MapReduceResult; +import com.google.appengine.tools.pipeline.JobInfo; +import com.google.appengine.tools.pipeline.NoSuchObjectException; +import com.google.common.collect.ImmutableMap; +import google.registry.request.Action; +import google.registry.request.HttpException.InternalServerErrorException; +import google.registry.request.JsonResponse; +import google.registry.request.Parameter; +import google.registry.request.auth.Auth; +import javax.inject.Inject; + +/** Action to poll the status of a mapreduce job. */ +@Action(path = PollMapreduceAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_ONLY) +public class PollMapreduceAction implements Runnable { + + public static final String PATH = "/_dr/task/pollMapreduce"; + + @Inject @Parameter("jobId") String jobId; + @Inject JsonResponse response; + @Inject PollMapreduceAction() {} + + @Override + public void run() { + JobInfo jobInfo; + try { + jobInfo = newPipelineService().getJobInfo(jobId); + } catch (NoSuchObjectException e) { + throw new InternalServerErrorException("Job not found: " + e); + } + ImmutableMap.Builder json = new ImmutableMap.Builder<>(); + json.put("state", jobInfo.getJobState().toString()); + if (jobInfo.getJobState() == JobInfo.State.COMPLETED_SUCCESSFULLY) { + json.put("output", ((MapReduceResult) jobInfo.getOutput()).getOutputResult().toString()); + } + response.setPayload(json.build()); + response.setStatus(SC_OK); + } +} diff --git a/java/google/registry/tools/server/ToolsServerModule.java b/java/google/registry/tools/server/ToolsServerModule.java index 662b65959..54a681324 100644 --- a/java/google/registry/tools/server/ToolsServerModule.java +++ b/java/google/registry/tools/server/ToolsServerModule.java @@ -95,4 +95,10 @@ public class ToolsServerModule { static String provideRawKeys(HttpServletRequest req) { return extractRequiredParameter(req, "rawKeys"); } + + @Provides + @Parameter("jobId") + String provideJobId(HttpServletRequest req) { + return extractRequiredParameter(req, "jobId"); + } } diff --git a/javatests/google/registry/module/tools/testdata/tools_routing.txt b/javatests/google/registry/module/tools/testdata/tools_routing.txt index af3f5918d..f19dd9b94 100644 --- a/javatests/google/registry/module/tools/testdata/tools_routing.txt +++ b/javatests/google/registry/module/tools/testdata/tools_routing.txt @@ -16,6 +16,7 @@ PATH CLASS METHODS OK AUTH /_dr/task/generateZoneFiles GenerateZoneFilesAction POST n INTERNAL,API APP ADMIN /_dr/task/killAllCommitLogs KillAllCommitLogsAction POST n INTERNAL APP IGNORED /_dr/task/killAllEppResources KillAllEppResourcesAction POST n INTERNAL APP IGNORED +/_dr/task/pollMapreduce PollMapreduceAction POST n INTERNAL APP IGNORED /_dr/task/refreshDnsForAllDomains RefreshDnsForAllDomainsAction GET n INTERNAL,API APP ADMIN /_dr/task/resaveAllEppResources ResaveAllEppResourcesAction GET n INTERNAL,API APP ADMIN /_dr/task/restoreCommitLogs RestoreCommitLogsAction POST y INTERNAL,API APP ADMIN diff --git a/javatests/google/registry/tools/server/BUILD b/javatests/google/registry/tools/server/BUILD index 298cb3c9d..bde2a9690 100644 --- a/javatests/google/registry/tools/server/BUILD +++ b/javatests/google/registry/tools/server/BUILD @@ -26,6 +26,7 @@ java_library( "@com_google_appengine_api_1_0_sdk//:testonly", "@com_google_appengine_tools_appengine_gcs_client", "@com_google_appengine_tools_appengine_mapreduce", + "@com_google_appengine_tools_appengine_pipeline", "@com_google_guava", "@com_google_re2j", "@com_google_truth", diff --git a/javatests/google/registry/tools/server/PollMapreduceActionTest.java b/javatests/google/registry/tools/server/PollMapreduceActionTest.java new file mode 100644 index 000000000..3f55c371e --- /dev/null +++ b/javatests/google/registry/tools/server/PollMapreduceActionTest.java @@ -0,0 +1,91 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.server; + +import static com.google.appengine.tools.pipeline.PipelineServiceFactory.newPipelineService; +import static com.google.common.truth.Truth.assertThat; + +import com.google.appengine.tools.mapreduce.Counters; +import com.google.appengine.tools.mapreduce.MapReduceResult; +import com.google.appengine.tools.pipeline.Job0; +import com.google.appengine.tools.pipeline.JobInfo.State; +import com.google.appengine.tools.pipeline.Value; +import google.registry.testing.FakeJsonResponse; +import google.registry.testing.mapreduce.MapreduceTestCase; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link PollMapreduceAction}.*/ +@RunWith(JUnit4.class) +public class PollMapreduceActionTest extends MapreduceTestCase { + + @Before + public void init() throws Exception { + action = new PollMapreduceAction(); + } + + @Test + public void testPollUntilSuccess() throws Exception { + action.jobId = newPipelineService().startNewPipeline(new SuccessfulJob()); + assertThat(poll()).containsExactly("state", State.RUNNING.toString()); + executeTasksUntilEmpty("default"); + assertThat(poll()).containsExactly( + "state", State.COMPLETED_SUCCESSFULLY.toString(), "output", "foobar"); + } + + @Test + public void testPollUntilFailure() throws Exception { + action.jobId = newPipelineService().startNewPipeline(new ThrowingJob()); + assertThat(poll()).containsExactly("state", State.RUNNING.toString()); + executeTasksUntilEmpty("default"); + assertThat(poll()).containsExactly("state", State.STOPPED_BY_ERROR.toString()); + } + + Map poll() { + action.response = new FakeJsonResponse(); + action.run(); + return ((FakeJsonResponse) action.response).getResponseMap(); + } + + /** A job that returns a string. */ + static class SuccessfulJob extends Job0> { + @Override + public Value> run() { + MapReduceResult result = + new MapReduceResult() { + @Override + public Counters getCounters() { + throw new UnsupportedOperationException(); + } + + @Override + public String getOutputResult() { + return "foobar"; + }}; + return immediate(result); + } + } + + /** A job that throws a RunTimeException when run. */ + static class ThrowingJob extends Job0 { + @Override + public Value run() { + throw new RuntimeException("expected"); + } + } +}