diff --git a/java/com/google/domain/registry/env/common/tools/WEB-INF/web.xml b/java/com/google/domain/registry/env/common/tools/WEB-INF/web.xml index 839a600c1..686ac6a9a 100644 --- a/java/com/google/domain/registry/env/common/tools/WEB-INF/web.xml +++ b/java/com/google/domain/registry/env/common/tools/WEB-INF/web.xml @@ -125,21 +125,6 @@ /_dr/task/deleteProberData - - - - Deletes entities in non-default namespaces. - - Annihilate Non-Default Namespaces Mapreduce - annihilate-non-default-namespaces - com.google.domain.registry.module.tools.ToolsServlet - 1 - - - annihilate-non-default-namespaces - /_dr/task/annihilateNonDefaultNamespaces - - diff --git a/java/com/google/domain/registry/module/tools/ToolsRequestComponent.java b/java/com/google/domain/registry/module/tools/ToolsRequestComponent.java index 0bfc9949a..d6b054338 100644 --- a/java/com/google/domain/registry/module/tools/ToolsRequestComponent.java +++ b/java/com/google/domain/registry/module/tools/ToolsRequestComponent.java @@ -37,7 +37,6 @@ import com.google.domain.registry.tools.server.ListTldsAction; import com.google.domain.registry.tools.server.ToolsServerModule; import com.google.domain.registry.tools.server.UpdatePremiumListAction; import com.google.domain.registry.tools.server.VerifyOteAction; -import com.google.domain.registry.tools.server.javascrap.AnnihilateNonDefaultNamespacesAction; import dagger.Subcomponent; @@ -51,7 +50,6 @@ import dagger.Subcomponent; ToolsServerModule.class, }) interface ToolsRequestComponent { - AnnihilateNonDefaultNamespacesAction annihilateNonDefaultNamespacesAction(); CreateGroupsAction createGroupsAction(); CreatePremiumListAction createPremiumListAction(); DeleteEntityAction deleteEntityAction(); diff --git a/java/com/google/domain/registry/tools/server/javascrap/AnnihilateNonDefaultNamespacesAction.java b/java/com/google/domain/registry/tools/server/javascrap/AnnihilateNonDefaultNamespacesAction.java deleted file mode 100644 index 6d7b6f3a9..000000000 --- a/java/com/google/domain/registry/tools/server/javascrap/AnnihilateNonDefaultNamespacesAction.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.domain.registry.tools.server.javascrap; - -import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService; -import static com.google.common.base.Preconditions.checkState; -import static com.google.domain.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN; -import static com.google.domain.registry.model.ofy.ObjectifyService.ofy; - -import com.google.appengine.api.NamespaceManager; -import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.datastore.Entities; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.Query; -import com.google.appengine.tools.mapreduce.Input; -import com.google.appengine.tools.mapreduce.Mapper; -import com.google.appengine.tools.mapreduce.inputs.DatastoreKeyInput; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; -import com.google.domain.registry.mapreduce.MapreduceAction; -import com.google.domain.registry.mapreduce.MapreduceRunner; -import com.google.domain.registry.mapreduce.inputs.ChunkingKeyInput; -import com.google.domain.registry.request.Action; -import com.google.domain.registry.request.Parameter; -import com.google.domain.registry.request.Response; -import com.google.domain.registry.util.FormattingLogger; -import com.google.domain.registry.util.PipelineUtils; - -import com.googlecode.objectify.VoidWork; - -import java.util.List; - -import javax.inject.Inject; - -/** - * A mapreduce that deletes all entities in all namespaces except for the default namespace. - */ -@Action(path = "/_dr/task/annihilateNonDefaultNamespaces") -public class AnnihilateNonDefaultNamespacesAction implements MapreduceAction { - - private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); - - @Inject DatastoreService datastoreService; - @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun; - @Inject MapreduceRunner mrRunner; - @Inject Response response; - @Inject AnnihilateNonDefaultNamespacesAction() {} - - @Override - public void run() { - response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner - .setJobName("Annihilate non-default namespaces") - .setModuleName("backend") - .runMapOnly( - new AnnihilateNonDefaultNamespacesMapper(isDryRun), - getInputs()))); - } - - /** Mapper to delete all entities in non-default namespaces. */ - public static class AnnihilateNonDefaultNamespacesMapper extends Mapper, Void, Void> { - - private static final long serialVersionUID = 8415923063881853727L; - private final boolean isDryRun; - - public AnnihilateNonDefaultNamespacesMapper(boolean isDryRun) { - this.isDryRun = isDryRun; - } - - @Override - public final void map(final List keys) { - if (isDryRun) { - logger.infofmt("Would delete these entities: %s", keys); - } else { - for (Key key : keys) { - // Additional safety check to prevent deleting real data. - checkState( - !Strings.isNullOrEmpty(key.getNamespace()), - "Will not delete key %s is in default namespace", - key); - } - ofy().transact(new VoidWork() { - @Override - public void vrun() { - getDatastoreService().delete(keys); - }}); - } - getContext().incrementCounter("entities deleted", keys.size()); - } - } - - private Iterable>> getInputs() { - ImmutableSet.Builder namespaces = new ImmutableSet.Builder<>(); - Query namespacesQuery = new Query(Entities.NAMESPACE_METADATA_KIND).setKeysOnly(); - for (Entity entity : datastoreService.prepare(namespacesQuery).asIterable()) { - // Don't delete anything in the default namespace! - if (!Strings.isNullOrEmpty(entity.getKey().getName())) { - namespaces.add(entity.getKey().getName()); - } - } - - ImmutableSet.Builder>> inputs = new ImmutableSet.Builder<>(); - for (String namespace : namespaces.build()) { - NamespaceManager.set(namespace); - ImmutableSet.Builder kindsBuilder = new ImmutableSet.Builder<>(); - Query kindsQuery = new Query(Entities.KIND_METADATA_KIND).setKeysOnly(); - for (Entity entity : datastoreService.prepare(kindsQuery).asIterable()) { - // Don't delete built-in kinds such as __Stat_* entities. - if (!entity.getKey().getName().startsWith("_")) { - kindsBuilder.add(entity.getKey().getName()); - } - } - ImmutableSet kinds = kindsBuilder.build(); - logger.infofmt("For namespace %s, found kinds: %s", namespace, kinds); - for (String kind : kinds) { - // Don't try to parallelize here, because Registry 1.0 entities are almost all in a single - // entity group. - inputs.add(new ChunkingKeyInput(new DatastoreKeyInput(kind, 1, namespace), 20)); - } - } - NamespaceManager.set(""); - return inputs.build(); - } -} diff --git a/javatests/com/google/domain/registry/tools/server/javascrap/AnnihilateNonDefaultNamespacesActionTest.java b/javatests/com/google/domain/registry/tools/server/javascrap/AnnihilateNonDefaultNamespacesActionTest.java deleted file mode 100644 index 994788970..000000000 --- a/javatests/com/google/domain/registry/tools/server/javascrap/AnnihilateNonDefaultNamespacesActionTest.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.domain.registry.tools.server.javascrap; - -import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService; -import static com.google.common.truth.Truth.assertThat; -import static com.google.domain.registry.model.ofy.ObjectifyService.ofy; -import static com.google.domain.registry.testing.DatastoreHelper.newRegistry; - -import com.google.appengine.api.NamespaceManager; -import com.google.common.base.Optional; -import com.google.common.collect.ImmutableSet; -import com.google.domain.registry.mapreduce.MapreduceRunner; -import com.google.domain.registry.model.ImmutableObject; -import com.google.domain.registry.model.domain.DomainApplication; -import com.google.domain.registry.model.host.HostResource; -import com.google.domain.registry.model.index.DomainApplicationIndex; -import com.google.domain.registry.model.registry.Registry; -import com.google.domain.registry.testing.DatastoreHelper; -import com.google.domain.registry.testing.FakeResponse; -import com.google.domain.registry.testing.mapreduce.MapreduceTestCase; - -import com.googlecode.objectify.Key; -import com.googlecode.objectify.Ref; -import com.googlecode.objectify.VoidWork; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link AnnihilateNonDefaultNamespacesAction}. */ -@RunWith(JUnit4.class) -public class AnnihilateNonDefaultNamespacesActionTest - extends MapreduceTestCase { - - @Before - public void init() { - action = new AnnihilateNonDefaultNamespacesAction(); - action.datastoreService = getDatastoreService(); - action.mrRunner = new MapreduceRunner(Optional.absent(), Optional.absent()); - action.response = new FakeResponse(); - } - - private void runMapreduce() throws Exception { - action.run(); - executeTasksUntilEmpty("mapreduce"); - } - - private void runTest(boolean shouldBeDeleted) throws Exception { - NamespaceManager.set("blah1"); - final DomainApplicationIndex toDelete1 = - DomainApplicationIndex.createWithSpecifiedReferences( - "foo.bar", ImmutableSet.of(Ref.create(Key.create(DomainApplication.class, 1L)))); - final Registry toDelete2 = newRegistry("bar", "BAR"); - ofy().transact(new VoidWork() { - @Override - public void vrun() { - ofy().save().entities(toDelete1, toDelete2).now(); - }}); - - NamespaceManager.set(""); - final HostResource dontDelete1 = DatastoreHelper.newHostResource("blah.test.bar"); - final Registry dontDelete2 = newRegistry("baz", "BAZ"); - ofy().transact(new VoidWork() { - @Override - public void vrun() { - ofy().save().entities(dontDelete1, dontDelete2).now(); - }}); - ofy().clearSessionCache(); - runMapreduce(); - assertThat(ofy().load().entity(dontDelete1).now()).isNotNull(); - assertThat(ofy().load().entity(dontDelete2).now()).isNotNull(); - - NamespaceManager.set("blah1"); - if (shouldBeDeleted) { - assertThat(ofy().load().entity(toDelete1).now()).isNull(); - assertThat(ofy().load().entity(toDelete2).now()).isNull(); - } else { - assertThat(ofy().load().entity(toDelete1).now()).isNotNull(); - assertThat(ofy().load().entity(toDelete2).now()).isNotNull(); - } - } - - @Test - public void test_deletesOnlyEntitesInNonDefaultNamespace() throws Exception { - runTest(true); - } - - @Test - public void test_dryRunDoesntDeleteAnything() throws Exception { - action.isDryRun = true; - runTest(false); - } -}