From e78ce42dd5ca07249348d4b86b7ca95db2ccac4e Mon Sep 17 00:00:00 2001 From: Pavlo Tkach <3469726+ptkach@users.noreply.github.com> Date: Fri, 26 Apr 2024 11:56:50 -0400 Subject: [PATCH] Add console DUM download (#2402) * Add console DUM download * Add console DUM download --- .../src/app/domains/domainList.component.html | 13 +- .../src/app/domains/domainList.component.scss | 9 ++ .../src/app/domains/domainList.component.ts | 2 +- .../registry/config/RegistryConfig.java | 11 ++ .../config/RegistryConfigSettings.java | 1 + .../registry/config/files/default-config.yaml | 3 + .../registry/module/RequestComponent.java | 3 + .../frontend/FrontendRequestComponent.java | 3 + .../google/registry/request/Response.java | 4 + .../google/registry/request/ResponseImpl.java | 6 + .../console/ConsoleDumDownloadAction.java | 128 ++++++++++++++++++ .../google/registry/testing/FakeResponse.java | 16 +++ .../console/ConsoleDumDownloadActionTest.java | 111 +++++++++++++++ .../module/frontend/frontend_routing.txt | 3 +- .../google/registry/module/routing.txt | 3 +- 15 files changed, 312 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/google/registry/ui/server/console/ConsoleDumDownloadAction.java create mode 100644 core/src/test/java/google/registry/ui/server/console/ConsoleDumDownloadActionTest.java diff --git a/console-webapp/src/app/domains/domainList.component.html b/console-webapp/src/app/domains/domainList.component.html index 1b3893a05..94bfcdcfb 100644 --- a/console-webapp/src/app/domains/domainList.component.html +++ b/console-webapp/src/app/domains/domainList.component.html @@ -1,6 +1,6 @@ -

Domains

+

Domains

@if (totalResults === 0) {

@@ -13,6 +13,17 @@ } @else if(isLoading) { } @else { + + download + Download domains (.csv) + Filter { diff --git a/core/src/main/java/google/registry/config/RegistryConfig.java b/core/src/main/java/google/registry/config/RegistryConfig.java index fec748db5..047e7683c 100644 --- a/core/src/main/java/google/registry/config/RegistryConfig.java +++ b/core/src/main/java/google/registry/config/RegistryConfig.java @@ -203,6 +203,17 @@ public final class RegistryConfig { return config.registrarConsole.announcementsEmailAddress; } + /** + * The DUM file name, used as a file name base for DUM csv file + * + * @see google.registry.ui.server.console.ConsoleDumDownloadAction + */ + @Provides + @Config("dumFileName") + public static String provideDumFileName(RegistryConfigSettings config) { + return config.registrarConsole.dumFileName; + } + /** * The contact phone number. Used in the "contact-us" section of the registrar console. * diff --git a/core/src/main/java/google/registry/config/RegistryConfigSettings.java b/core/src/main/java/google/registry/config/RegistryConfigSettings.java index eab156452..a63c020b8 100644 --- a/core/src/main/java/google/registry/config/RegistryConfigSettings.java +++ b/core/src/main/java/google/registry/config/RegistryConfigSettings.java @@ -185,6 +185,7 @@ public class RegistryConfigSettings { /** Configuration for the web-based registrar console. */ public static class RegistrarConsole { + public String dumFileName; public String logoFilename; public String supportPhoneNumber; public String supportEmailAddress; diff --git a/core/src/main/java/google/registry/config/files/default-config.yaml b/core/src/main/java/google/registry/config/files/default-config.yaml index 7b85a803e..a96a92f47 100644 --- a/core/src/main/java/google/registry/config/files/default-config.yaml +++ b/core/src/main/java/google/registry/config/files/default-config.yaml @@ -396,6 +396,9 @@ rde: sshIdentityEmailAddress: rde@example.com registrarConsole: + # DUM download file name, excluding the extension + dumFileName: dum_file_name + # Filename of the logo to use in the header of the console. This filename is # relative to ui/assets/images/ logoFilename: logo.png diff --git a/core/src/main/java/google/registry/module/RequestComponent.java b/core/src/main/java/google/registry/module/RequestComponent.java index 7645e835f..f662c523f 100644 --- a/core/src/main/java/google/registry/module/RequestComponent.java +++ b/core/src/main/java/google/registry/module/RequestComponent.java @@ -110,6 +110,7 @@ import google.registry.tools.server.ToolsServerModule; import google.registry.tools.server.VerifyOteAction; import google.registry.ui.server.console.ConsoleDomainGetAction; import google.registry.ui.server.console.ConsoleDomainListAction; +import google.registry.ui.server.console.ConsoleDumDownloadAction; import google.registry.ui.server.console.ConsoleEppPasswordAction; import google.registry.ui.server.console.ConsoleUserDataAction; import google.registry.ui.server.console.RegistrarsAction; @@ -189,6 +190,8 @@ interface RequestComponent { ConsoleUserDataAction consoleUserDataAction(); + ConsoleDumDownloadAction ConsoleDumDownloadAction(); + ContactAction contactAction(); CopyDetailReportsAction copyDetailReportAction(); diff --git a/core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java b/core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java index bf40ef474..e0f15221f 100644 --- a/core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java +++ b/core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java @@ -27,6 +27,7 @@ import google.registry.request.RequestModule; import google.registry.request.RequestScope; import google.registry.ui.server.console.ConsoleDomainGetAction; import google.registry.ui.server.console.ConsoleDomainListAction; +import google.registry.ui.server.console.ConsoleDumDownloadAction; import google.registry.ui.server.console.ConsoleEppPasswordAction; import google.registry.ui.server.console.ConsoleUserDataAction; import google.registry.ui.server.console.RegistrarsAction; @@ -67,6 +68,8 @@ public interface FrontendRequestComponent { ConsoleUserDataAction consoleUserDataAction(); + ConsoleDumDownloadAction ConsoleDumDownloadAction(); + ContactAction contactAction(); EppTlsAction eppTlsAction(); diff --git a/core/src/main/java/google/registry/request/Response.java b/core/src/main/java/google/registry/request/Response.java index d39c0abe8..e0d5accb2 100644 --- a/core/src/main/java/google/registry/request/Response.java +++ b/core/src/main/java/google/registry/request/Response.java @@ -17,6 +17,8 @@ package google.registry.request; import com.google.common.net.MediaType; import jakarta.servlet.http.Cookie; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; import org.joda.time.DateTime; /** @@ -59,4 +61,6 @@ public interface Response { * @see HttpServletResponse#addCookie(Cookie) */ void addCookie(Cookie cookie); + + PrintWriter getWriter() throws IOException; } diff --git a/core/src/main/java/google/registry/request/ResponseImpl.java b/core/src/main/java/google/registry/request/ResponseImpl.java index ec2c537b5..864ee920e 100644 --- a/core/src/main/java/google/registry/request/ResponseImpl.java +++ b/core/src/main/java/google/registry/request/ResponseImpl.java @@ -18,6 +18,7 @@ import com.google.common.net.MediaType; import jakarta.servlet.http.Cookie; import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; +import java.io.PrintWriter; import javax.inject.Inject; import org.joda.time.DateTime; @@ -64,4 +65,9 @@ public final class ResponseImpl implements Response { public void addCookie(Cookie cookie) { rsp.addCookie(cookie); } + + @Override + public PrintWriter getWriter() throws IOException { + return rsp.getWriter(); + } } diff --git a/core/src/main/java/google/registry/ui/server/console/ConsoleDumDownloadAction.java b/core/src/main/java/google/registry/ui/server/console/ConsoleDumDownloadAction.java new file mode 100644 index 000000000..23ed07f14 --- /dev/null +++ b/core/src/main/java/google/registry/ui/server/console/ConsoleDumDownloadAction.java @@ -0,0 +1,128 @@ +// Copyright 2024 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.ui.server.console; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.persistence.transaction.TransactionManagerFactory.tm; +import static google.registry.request.Action.Method.GET; +import static org.joda.time.DateTimeZone.UTC; + +import com.google.common.collect.ImmutableList; +import com.google.common.flogger.FluentLogger; +import com.google.common.net.MediaType; +import google.registry.config.RegistryConfig.Config; +import google.registry.model.console.ConsolePermission; +import google.registry.model.console.User; +import google.registry.request.Action; +import google.registry.request.Parameter; +import google.registry.request.auth.Auth; +import google.registry.ui.server.registrar.ConsoleApiParams; +import google.registry.util.Clock; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import javax.inject.Inject; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.joda.time.DateTime; + +@Action( + service = Action.Service.DEFAULT, + path = ConsoleDumDownloadAction.PATH, + method = {GET}, + auth = Auth.AUTH_PUBLIC_LOGGED_IN) +public class ConsoleDumDownloadAction extends ConsoleApiAction { + + private static final String SQL_TEMPLATE = + """ + SELECT CONCAT( + d.domain_name,',',d.creation_time,',',d.registration_expiration_time,',',d.statuses + ) AS result FROM "Domain" d + WHERE d.current_sponsor_registrar_id = :registrarId + AND d.deletion_time > ':now' + AND d.creation_time <= ':now'; + """; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + public static final String PATH = "/console-api/dum-download"; + private Clock clock; + private final String registrarId; + private final String dumFileName; + + @Inject + public ConsoleDumDownloadAction( + Clock clock, + ConsoleApiParams consoleApiParams, + @Parameter("registrarId") String registrarId, + @Config("dumFileName") String dumFileName) { + super(consoleApiParams); + this.registrarId = registrarId; + this.clock = clock; + this.dumFileName = dumFileName; + } + + @Override + protected void getHandler(User user) { + if (!user.getUserRoles().hasPermission(registrarId, ConsolePermission.DOWNLOAD_DOMAINS)) { + consoleApiParams.response().setStatus(HttpServletResponse.SC_FORBIDDEN); + return; + } + + consoleApiParams.response().setContentType(MediaType.CSV_UTF_8); + consoleApiParams + .response() + .setHeader( + "Content-Disposition", String.format("attachment; filename=%s.csv", dumFileName)); + consoleApiParams + .response() + .setHeader("Cache-Control", "max-age=86400"); // 86400 seconds = 1 day + consoleApiParams + .response() + .setDateHeader("Expires", DateTime.now(UTC).withTimeAtStartOfDay().plusDays(1)); + + try (var writer = consoleApiParams.response().getWriter()) { + CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT); + writeCsv(csvPrinter); + } catch (IOException e) { + logger.atWarning().withCause(e).log( + String.format("Failed to create DUM csv for %s", registrarId)); + consoleApiParams.response().setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + consoleApiParams.response().setStatus(HttpServletResponse.SC_OK); + } + + private void writeCsv(CSVPrinter printer) throws IOException { + String sql = SQL_TEMPLATE.replaceAll(":now", clock.nowUtc().toString()); + + // We deliberately don't want to use ImmutableList.copyOf because underlying list may contain + // large amount of records and that will degrade performance. + List queryResult = + tm().transact( + () -> + tm().getEntityManager() + .createNativeQuery(sql) + .setParameter("registrarId", registrarId) + .setHint("org.hibernate.fetchSize", 1000) + .getResultList()); + + ImmutableList formattedRecords = + queryResult.stream().map(r -> r.split(",")).collect(toImmutableList()); + printer.printRecord( + ImmutableList.of("Domain Name", "Creation Time", "Expiration Time", "Domain Statuses")); + printer.printRecords(formattedRecords); + } +} diff --git a/core/src/test/java/google/registry/testing/FakeResponse.java b/core/src/test/java/google/registry/testing/FakeResponse.java index 16a23d5fb..16689ad62 100644 --- a/core/src/test/java/google/registry/testing/FakeResponse.java +++ b/core/src/test/java/google/registry/testing/FakeResponse.java @@ -23,6 +23,9 @@ import com.google.common.base.Throwables; import com.google.common.net.MediaType; import google.registry.request.Response; import jakarta.servlet.http.Cookie; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -39,6 +42,9 @@ public final class FakeResponse implements Response { private boolean wasMutuallyExclusiveResponseSet; private String lastResponseStackTrace; + private final StringWriter writer = new StringWriter(); + private PrintWriter printWriter = new PrintWriter(writer); + private ArrayList cookies = new ArrayList<>(); public int getStatus() { @@ -57,6 +63,10 @@ public final class FakeResponse implements Response { return unmodifiableMap(headers); } + public StringWriter getStringWriter() { + return writer; + } + @Override public void setStatus(int status) { checkArgument(status >= 100); @@ -93,6 +103,11 @@ public final class FakeResponse implements Response { cookies.add(cookie); } + @Override + public PrintWriter getWriter() throws IOException { + return printWriter; + } + public List getCookies() { return cookies; } @@ -113,4 +128,5 @@ public final class FakeResponse implements Response { return Throwables.getStackTraceAsString(e); } } + } diff --git a/core/src/test/java/google/registry/ui/server/console/ConsoleDumDownloadActionTest.java b/core/src/test/java/google/registry/ui/server/console/ConsoleDumDownloadActionTest.java new file mode 100644 index 000000000..6c73183ef --- /dev/null +++ b/core/src/test/java/google/registry/ui/server/console/ConsoleDumDownloadActionTest.java @@ -0,0 +1,111 @@ +// Copyright 2024 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.ui.server.console; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.testing.DatabaseHelper.createTld; +import static org.mockito.Mockito.when; + +import com.google.api.client.http.HttpStatusCodes; +import com.google.common.collect.ImmutableList; +import com.google.gson.Gson; +import google.registry.model.console.GlobalRole; +import google.registry.model.console.User; +import google.registry.model.console.UserRoles; +import google.registry.persistence.transaction.JpaTestExtensions; +import google.registry.request.Action; +import google.registry.request.auth.AuthResult; +import google.registry.request.auth.UserAuthInfo; +import google.registry.testing.DatabaseHelper; +import google.registry.testing.FakeClock; +import google.registry.testing.FakeConsoleApiParams; +import google.registry.testing.FakeResponse; +import google.registry.tools.GsonUtils; +import google.registry.ui.server.registrar.ConsoleApiParams; +import java.io.IOException; +import java.util.Optional; +import org.joda.time.DateTime; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +class ConsoleDumDownloadActionTest { + + private static final Gson GSON = GsonUtils.provideGson(); + + private final FakeClock clock = new FakeClock(DateTime.parse("2024-04-15T00:00:00.000Z")); + + private ConsoleApiParams consoleApiParams; + + @RegisterExtension + final JpaTestExtensions.JpaIntegrationTestExtension jpa = + new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension(); + + @BeforeEach + void beforeEach() { + createTld("tld"); + for (int i = 0; i < 3; i++) { + DatabaseHelper.persistActiveDomain( + i + "exists.tld", clock.nowUtc(), clock.nowUtc().plusDays(300)); + clock.advanceOneMilli(); + } + DatabaseHelper.persistDeletedDomain("deleted.tld", clock.nowUtc().minusDays(1)); + } + + @Test + void testSuccess_returnsCorrectDomains() throws IOException { + User user = + new User.Builder() + .setEmailAddress("email@email.com") + .setUserRoles(new UserRoles.Builder().setGlobalRole(GlobalRole.FTE).build()) + .build(); + + AuthResult authResult = AuthResult.createUser(UserAuthInfo.create(user)); + ConsoleDumDownloadAction action = createAction(Optional.of(authResult)); + action.run(); + ImmutableList expected = + ImmutableList.of( + "Domain Name,Creation Time,Expiration Time,Domain Statuses", + "2exists.tld,2024-04-15 00:00:00.002+00,2025-02-09 00:00:00.002+00,{INACTIVE}", + "1exists.tld,2024-04-15 00:00:00.001+00,2025-02-09 00:00:00.001+00,{INACTIVE}", + "0exists.tld,2024-04-15 00:00:00+00,2025-02-09 00:00:00+00,{INACTIVE}"); + FakeResponse response = (FakeResponse) consoleApiParams.response(); + assertThat(response.getStatus()).isEqualTo(HttpStatusCodes.STATUS_CODE_OK); + ImmutableList actual = + ImmutableList.copyOf(response.getStringWriter().toString().split("\r\n")); + assertThat(actual).containsExactlyElementsIn(expected); + } + + @Test + void testFailure_forbidden() { + UserRoles userRoles = + new UserRoles.Builder().setGlobalRole(GlobalRole.NONE).setIsAdmin(false).build(); + + User user = + new User.Builder().setEmailAddress("email@email.com").setUserRoles(userRoles).build(); + + AuthResult authResult = AuthResult.createUser(UserAuthInfo.create(user)); + ConsoleDumDownloadAction action = createAction(Optional.of(authResult)); + action.run(); + assertThat(((FakeResponse) consoleApiParams.response()).getStatus()) + .isEqualTo(HttpStatusCodes.STATUS_CODE_FORBIDDEN); + } + + private ConsoleDumDownloadAction createAction(Optional maybeAuthResult) { + consoleApiParams = FakeConsoleApiParams.get(maybeAuthResult); + when(consoleApiParams.request().getMethod()).thenReturn(Action.Method.GET.toString()); + return new ConsoleDumDownloadAction(clock, consoleApiParams, "TheRegistrar", "test_name"); + } +} diff --git a/core/src/test/resources/google/registry/module/frontend/frontend_routing.txt b/core/src/test/resources/google/registry/module/frontend/frontend_routing.txt index c708534cf..8951d7077 100644 --- a/core/src/test/resources/google/registry/module/frontend/frontend_routing.txt +++ b/core/src/test/resources/google/registry/module/frontend/frontend_routing.txt @@ -2,6 +2,7 @@ PATH CLASS METHODS OK AUT /_dr/epp EppTlsAction POST n API APP ADMIN /console-api/domain ConsoleDomainGetAction GET n API,LEGACY USER PUBLIC /console-api/domain-list ConsoleDomainListAction GET n API,LEGACY USER PUBLIC +/console-api/dum-download ConsoleDumDownloadAction GET n API,LEGACY USER PUBLIC /console-api/eppPassword ConsoleEppPasswordAction POST n API,LEGACY USER PUBLIC /console-api/registrars RegistrarsAction GET,POST n API,LEGACY USER PUBLIC /console-api/settings/contacts ContactAction GET,POST n API,LEGACY USER PUBLIC @@ -15,4 +16,4 @@ PATH CLASS METHODS OK AUT /registrar-settings RegistrarSettingsAction POST n API,LEGACY USER PUBLIC /registry-lock-get RegistryLockGetAction GET n API,LEGACY USER PUBLIC /registry-lock-post RegistryLockPostAction POST n API,LEGACY USER PUBLIC -/registry-lock-verify RegistryLockVerifyAction GET n API,LEGACY NONE PUBLIC \ No newline at end of file +/registry-lock-verify RegistryLockVerifyAction GET n API,LEGACY NONE PUBLIC diff --git a/core/src/test/resources/google/registry/module/routing.txt b/core/src/test/resources/google/registry/module/routing.txt index 748e1fe3a..376ebb319 100644 --- a/core/src/test/resources/google/registry/module/routing.txt +++ b/core/src/test/resources/google/registry/module/routing.txt @@ -57,6 +57,7 @@ PATH CLASS /check CheckApiAction GET n API NONE PUBLIC /console-api/domain ConsoleDomainGetAction GET n API,LEGACY USER PUBLIC /console-api/domain-list ConsoleDomainListAction GET n API,LEGACY USER PUBLIC +/console-api/dum-download ConsoleDumDownloadAction GET n API,LEGACY USER PUBLIC /console-api/eppPassword ConsoleEppPasswordAction POST n API,LEGACY USER PUBLIC /console-api/registrars RegistrarsAction GET,POST n API,LEGACY USER PUBLIC /console-api/settings/contacts ContactAction GET,POST n API,LEGACY USER PUBLIC @@ -80,4 +81,4 @@ PATH CLASS /registry-lock-get RegistryLockGetAction GET n API,LEGACY USER PUBLIC /registry-lock-post RegistryLockPostAction POST n API,LEGACY USER PUBLIC /registry-lock-verify RegistryLockVerifyAction GET n API,LEGACY NONE PUBLIC -/whois/(*) WhoisHttpAction GET n API NONE PUBLIC \ No newline at end of file +/whois/(*) WhoisHttpAction GET n API NONE PUBLIC