From 5c9274770b6cae937b4ec8a108c22fa76a963f27 Mon Sep 17 00:00:00 2001 From: guyben Date: Tue, 19 Feb 2019 07:52:46 -0800 Subject: [PATCH] Upload task results to GCS This uploads all the report files to GCS, and also creates a cover page that summarises all the task states. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=234601015 --- gradle/build.gradle | 19 +- gradle/buildSrc/build.gradle | 14 + .../gradle/plugin/CoverPageGenerator.java | 207 +++++++++++++ .../gradle/plugin/GcsPluginUtils.java | 206 +++++++++++++ .../gradle/plugin/GcsReportUploader.java | 219 ++++++++++++- .../plugin/GcsReportUploaderPlugin.java | 4 +- .../registry/gradle/plugin/ProjectData.java | 169 ++++++++++ .../registry/gradle/plugin/css/style.css | 27 ++ .../registry/gradle/plugin/soy/coverpage.soy | 107 +++++++ .../gradle/plugin/CoverPageGeneratorTest.java | 288 +++++++++++++++++ .../gradle/plugin/GcsPluginUtilsTest.java | 291 ++++++++++++++++++ gradle/dependencies.gradle | 1 + gradle/gradle.properties | 3 + 13 files changed, 1538 insertions(+), 17 deletions(-) create mode 100644 gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java create mode 100644 gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java create mode 100644 gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java create mode 100644 gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css create mode 100644 gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy create mode 100644 gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java create mode 100644 gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java diff --git a/gradle/build.gradle b/gradle/build.gradle index d540471d2..6313237a2 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -26,7 +26,24 @@ plugins { apply plugin: google.registry.gradle.plugin.GcsReportUploaderPlugin gcsReportUploader { - bucket = 'my-bucket' + // Set the bucket here to upload build results to a GCS bucket. + // e.g. -P gcsBucket=domain-registry-alpha-build-result-test + // + // If no bucket it set - the uploading will be skipped. + bucket = gcsBucket + + // The location of the file containing the OAuth2 Google Cloud credentials. + // + // The file can contain a Service Account key file in JSON format from the + // Google Developers Console or a stored user credential using the format + // supported by the Cloud SDK. + // + // If no file is given - the default credentials are used. + credentialsFile = gcsCredentialsFile + + // If set to 'yes', each file will be uploaded to GCS in a separate thread. + // This is MUCH faster. + multithreadedUpload = gcsMultithreadedUpload } apply from: 'dependencies.gradle' diff --git a/gradle/buildSrc/build.gradle b/gradle/buildSrc/build.gradle index e27d22cab..bf8c78c3f 100644 --- a/gradle/buildSrc/build.gradle +++ b/gradle/buildSrc/build.gradle @@ -17,5 +17,19 @@ apply from: '../dependencies.gradle' dependencies { def deps = dependencyMap compile deps['com.google.guava:guava'] + compile deps['com.google.auto.value:auto-value-annotations'] + compile deps['com.google.cloud:google-cloud-storage'] + compile deps['org.apache.commons:commons-text'] + compile deps['com.google.auth:google-auth-library-credentials'] + compile deps['com.google.template:soy'] + annotationProcessor deps['com.google.auto.value:auto-value'] testCompile deps['com.google.truth:truth'] + testCompile deps['com.google.truth.extensions:truth-java8-extension'] + testCompile deps['org.mockito:mockito-all'] +} + +gradle.projectsEvaluated { + tasks.withType(JavaCompile) { + options.compilerArgs << "-Xlint:unchecked" + } } diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java new file mode 100644 index 000000000..f71c60607 --- /dev/null +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java @@ -0,0 +1,207 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.gradle.plugin; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap; +import static com.google.common.io.Resources.getResource; +import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.template.soy.SoyFileSet; +import com.google.template.soy.tofu.SoyTofu; +import google.registry.gradle.plugin.ProjectData.TaskData; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Creates the files for a web-page summary of a given {@Link ProjectData}. + * + *

The main job of this class is rendering a tailored cover page that includes information about + * the project and any task that ran. + * + *

It returns all the files that need uploading for the cover page to work. This includes any + * report and log files linked to in the ProjectData, as well as a cover page (and associated + * resources such as CSS files). + */ +final class CoverPageGenerator { + + /** List of all resource files that will be uploaded as-is. */ + private static final ImmutableSet STATIC_RESOURCE_FILES = + ImmutableSet.of(Paths.get("css", "style.css")); + + private final ProjectData projectData; + private final ImmutableSetMultimap tasksByState; + /** + * The compiled SOY files. + * + *

Will be generated only when actually needed, because it takes a while to compile and we + * don't want that to happen unless we actually use it. + */ + private SoyTofu tofu = null; + + CoverPageGenerator(ProjectData projectData) { + this.projectData = projectData; + this.tasksByState = + projectData.tasks().stream().collect(toImmutableSetMultimap(TaskData::state, task -> task)); + } + + /** + * The (relative) entry point for the cover page. + * + *

A file with this relative path is guaranteed to be returned from {@link #getFilesToUpload}, + * and a browser pointing to this page will have access to all the data generated by {@link + * #getFilesToUpload}. + */ + Path getEntryPoint() { + return Paths.get("index.html"); + } + + /** + * Returns all the files that need uploading for the cover page to work. + * + *

This includes all the report files as well, to make sure that the link works. + */ + ImmutableMap> getFilesToUpload() { + ImmutableMap.Builder> builder = new ImmutableMap.Builder<>(); + // Add all the static resource pages + STATIC_RESOURCE_FILES.stream().forEach(file -> builder.put(file, resourceLoader(file))); + // Create the cover page + // Note that the ByteArraySupplier here is lazy - the createCoverPage function is only called + // when the resulting Supplier's get function is called. + builder.put(getEntryPoint(), toByteArraySupplier(this::createCoverPage)); + // Add all the files from the tasks + tasksByState.values().stream() + .flatMap(task -> task.reports().values().stream()) + .forEach(reportFiles -> builder.putAll(reportFiles.files())); + // Add the logs of every test + tasksByState.values().stream() + .filter(task -> task.log().isPresent()) + .forEach(task -> builder.put(getLogPath(task), task.log().get())); + + return builder.build(); + } + + /** Renders the cover page. */ + private String createCoverPage() { + return getTofu() + .newRenderer("google.registry.gradle.plugin.coverPage") + .setData(getSoyData()) + .render(); + } + + /** Converts the projectData and all taskData into all the data the soy template needs. */ + private ImmutableMap getSoyData() { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + + TaskData.State state = + tasksByState.containsKey(TaskData.State.FAILURE) + ? TaskData.State.FAILURE + : TaskData.State.SUCCESS; + String title = + state != TaskData.State.FAILURE + ? "Success!" + : "Failed: " + + tasksByState.get(state).stream() + .map(TaskData::uniqueName) + .collect(Collectors.joining(", ")); + + builder.put("projectState", state.toString()); + builder.put("title", title); + builder.put("cssFiles", ImmutableSet.of("css/style.css")); + builder.put("invocation", getInvocation()); + builder.put("tasksByState", getTasksByStateSoyData()); + return builder.build(); + } + + /** + * Returns a soy-friendly map from the TaskData.State to the task itslef. + * + *

The key order in the resulting map is always the same (the order from the enum definition) + * no matter the key order in the original tasksByState map. + */ + private ImmutableMap getTasksByStateSoyData() { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + + // We go over the States in the order they are defined rather than the order in which they + // happen to be in the tasksByState Map. + // + // That way we guarantee a consistent order. + for (TaskData.State state : TaskData.State.values()) { + builder.put( + state.toString(), + tasksByState.get(state).stream() + .map(task -> taskDataToSoy(task)) + .collect(toImmutableList())); + } + + return builder.build(); + } + + /** returns a soy-friendly version of the given task data. */ + static ImmutableMap taskDataToSoy(TaskData task) { + return new ImmutableMap.Builder() + .put("uniqueName", task.uniqueName()) + .put("description", task.description()) + .put("log", task.log().isPresent() ? getLogPath(task).toString() : "") + .put( + "reports", + task.reports().entrySet().stream() + .collect( + toImmutableMap( + entry -> entry.getKey(), + entry -> + entry.getValue().files().isEmpty() + ? "" + : entry.getValue().entryPoint().toString()))) + .build(); + } + + private String getInvocation() { + StringBuilder builder = new StringBuilder(); + builder.append("./gradlew"); + projectData.tasksRequested().forEach(task -> builder.append(" ").append(task)); + projectData + .projectProperties() + .forEach((key, value) -> builder.append(String.format(" -P %s=%s", key, value))); + return builder.toString(); + } + + /** Returns a lazily created soy renderer */ + private SoyTofu getTofu() { + if (tofu == null) { + tofu = + SoyFileSet.builder() + .add(getResource(CoverPageGenerator.class, "soy/coverpage.soy")) + .build() + .compileToTofu(); + } + return tofu; + } + + private static Path getLogPath(TaskData task) { + // TODO(guyben):escape uniqueName to guarantee legal file name. + return Paths.get("logs", task.uniqueName() + ".log"); + } + + private static Supplier resourceLoader(Path path) { + return toByteArraySupplier(getResource(CoverPageGenerator.class, path.toString())); + } +} diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java new file mode 100644 index 000000000..489c91f0c --- /dev/null +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java @@ -0,0 +1,206 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.gradle.plugin; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Streams; +import com.google.common.io.Files; +import com.google.common.io.Resources; +import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URL; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** Utility functions used in the GCS plugin. */ +final class GcsPluginUtils { + + private static final ImmutableMap EXTENSION_TO_CONTENT_TYPE = + new ImmutableMap.Builder() + .put("html", "text/html") + .put("htm", "text/html") + .put("log", "text/plain") + .put("txt", "text/plain") + .put("css", "text/css") + .put("xml", "text/xml") + .put("zip", "application/zip") + .put("js", "text/javascript") + .build(); + + private static final String DEFAULT_CONTENT_TYPE = "application/octet-stream"; + + static Path toNormalizedPath(File file) { + return file.toPath().toAbsolutePath().normalize(); + } + + static String getContentType(String fileName) { + return EXTENSION_TO_CONTENT_TYPE.getOrDefault( + Files.getFileExtension(fileName), DEFAULT_CONTENT_TYPE); + } + + static void uploadFileToGcs( + Storage storage, String bucket, Path path, Supplier dataSupplier) { + String filename = path.toString(); + storage.create( + BlobInfo.newBuilder(bucket, filename).setContentType(getContentType(filename)).build(), + dataSupplier.get()); + } + + static void uploadFilesToGcsMultithread( + Storage storage, String bucket, Path folder, Map> files) { + ImmutableMap.Builder threads = new ImmutableMap.Builder<>(); + files.forEach( + (path, dataSupplier) -> { + Thread thread = + new Thread( + () -> uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier)); + thread.start(); + threads.put(path, thread); + }); + threads + .build() + .forEach( + (path, thread) -> { + try { + thread.join(); + } catch (InterruptedException e) { + System.out.format("Upload of %s interrupted", path); + } + }); + } + + static Supplier toByteArraySupplier(String data) { + return () -> data.getBytes(UTF_8); + } + + static Supplier toByteArraySupplier(Supplier dataSupplier) { + return () -> dataSupplier.get().getBytes(UTF_8); + } + + static Supplier toByteArraySupplier(File file) { + return () -> { + try { + return Files.toByteArray(file); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + static Supplier toByteArraySupplier(URL url) { + return () -> { + try { + return Resources.toByteArray(url); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + /** + * Reads all the files generated by a Report into a ReportFiles object. + * + *

Every ReportFiles must have a single link "entry point" that gives users access to all the + * files. If the report generated just one file - we will just link to that file. + * + *

However, if the report generated more than one file - the only thing we can safely do is to + * zip all the files and link to the zip file. + * + *

As an alternative to using a zip file, we allow the caller to supply an optional "entry + * point" file that will link to all the other files. If that file is given and is "appropriate" + * (exists and is in the correct location) - we will upload all the report files "as is" and link + * to the entry file. + * + * @param destination the location of the output. Either a file or a directory. If a directory - + * then all the files inside that directory are the outputs we're looking for. + * @param entryPointHint If present - a hint to what the entry point to this directory tree is. + * Will only be used if all of the following apply: (a) {@code + * destination.isDirectory()==true}, (b) there are 2 or more files in the {@code destination} + * directory, and (c) {@code entryPointHint.get()} is one of the files nested inside of the + * {@code destination} directory. + */ + static ReportFiles createReportFiles( + File destination, Optional entryPointHint, Path rootDir) { + + Path destinationPath = rootDir.relativize(toNormalizedPath(destination)); + + if (destination.isFile()) { + // The destination is a single file - find its root, and add this single file to the + // ReportFiles. + return ReportFiles.create( + ImmutableMap.of(destinationPath, toByteArraySupplier(destination)), destinationPath); + } + + if (!destination.isDirectory()) { + // This isn't a file nor a directory - so it doesn't exist! Return empty ReportFiles + return ReportFiles.create(ImmutableMap.of(), destinationPath); + } + + // The destination is a directory - find all the actual files first + ImmutableMap> files = + Streams.stream(Files.fileTraverser().depthFirstPreOrder(destination)) + .filter(File::isFile) + .collect( + toImmutableMap( + file -> rootDir.relativize(toNormalizedPath(file)), + file -> toByteArraySupplier(file))); + + if (files.isEmpty()) { + // The directory exists, but is empty. Return empty ReportFiles + return ReportFiles.create(ImmutableMap.of(), destinationPath); + } + + if (files.size() == 1) { + // We got a directory, but it only has a single file. We can link to that. + return ReportFiles.create(files, getOnlyElement(files.keySet())); + } + + // There are multiple files in the report! We need to check the entryPointHint + Optional entryPointPath = + entryPointHint.map(file -> rootDir.relativize(toNormalizedPath(file))); + + if (entryPointPath.isPresent() && files.containsKey(entryPointPath.get())) { + // We were given the entry point! Use it! + return ReportFiles.create(files, entryPointPath.get()); + } + + // We weren't given an appropriate entry point. But we still need a single link to all this data + // - so we'll zip it and just host a single file. + // + // TODO(guyben):the zip part is still unimplemented, but what we'll want to do is this: + // Supplier zippedSupplier = createZippedByteArraySupplier(files); + // Path zipFilePath = rootFolder.resolve(rootFolder.getFileName().toString() + ".zip"); + // return ReportFiles.create(ImmutableMap.of(zipFilePath, zippedSupplier), zipFilePath); + Path unimplementedPath = destinationPath.resolve("unimplemented.txt"); + String content = + "Zip files are currently unimplemented. Files:\n" + + files.keySet().stream().map(Object::toString).collect(Collectors.joining("\n")); + return ReportFiles.create( + ImmutableMap.of(unimplementedPath, toByteArraySupplier(content)), unimplementedPath); + } + + private GcsPluginUtils() {} +} diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java index 04f3f287f..4d06eaf5a 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java @@ -14,9 +14,35 @@ package google.registry.gradle.plugin; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Strings.isNullOrEmpty; +import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles; +import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; +import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath; +import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs; +import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread; + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.common.collect.ImmutableMap; +import google.registry.gradle.plugin.ProjectData.TaskData; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.SecureRandom; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Optional; +import java.util.function.Supplier; import org.gradle.api.DefaultTask; +import org.gradle.api.Project; import org.gradle.api.Task; +import org.gradle.api.reporting.DirectoryReport; import org.gradle.api.reporting.Report; import org.gradle.api.reporting.ReportContainer; import org.gradle.api.reporting.Reporting; @@ -24,33 +50,200 @@ import org.gradle.api.tasks.TaskAction; /** * A task that uploads the Reports generated by other tasks to GCS. - * - *

TODO:implement it. */ public class GcsReportUploader extends DefaultTask { - private final ArrayList reportingTasks = new ArrayList<>(); + private static final SecureRandom secureRandom = new SecureRandom(); + + private final ArrayList tasks = new ArrayList<>(); + private final HashMap logs = new HashMap<>(); + private Project project; private String bucket = null; + private String credentialsFile = null; + private String multithreadedUpload = null; public void setBucket(String bucket) { this.bucket = bucket; } + public void setCredentialsFile(String credentialsFile) { + this.credentialsFile = credentialsFile; + } + + public void setMultithreadedUpload(String multithreadedUpload) { + this.multithreadedUpload = multithreadedUpload; + } + + /** Converts the given Gradle Project into a ProjectData. */ + private ProjectData createProjectData() { + ProjectData.Builder builder = + ProjectData.builder() + .setName(project.getPath() + project.getName()) + .setDescription( + Optional.ofNullable(project.getDescription()).orElse("[No description available]")) + .setGradleVersion(project.getGradle().getGradleVersion()) + .setProjectProperties(project.getGradle().getStartParameter().getProjectProperties()) + .setSystemProperties(project.getGradle().getStartParameter().getSystemPropertiesArgs()) + .setTasksRequested(project.getGradle().getStartParameter().getTaskNames()); + + Path rootDir = toNormalizedPath(project.getRootDir()); + tasks.stream() + .filter(task -> task.getState().getExecuted() || task.getState().getUpToDate()) + .map(task -> createTaskData(task, rootDir)) + .forEach(builder.tasksBuilder()::add); + return builder.build(); + } + + /** + * Converts a Gradle Task into a TaskData. + * + * @param rootDir the root directory of the main Project - used to get the relative path of any + * Task files. + */ + private TaskData createTaskData(Task task, Path rootDir) { + TaskData.State state = + task.getState().getFailure() != null + ? TaskData.State.FAILURE + : task.getState().getUpToDate() ? TaskData.State.UP_TO_DATE : TaskData.State.SUCCESS; + String log = logs.get(task.getPath()).toString(); + + TaskData.Builder builder = + TaskData.builder() + .setState(state) + .setUniqueName(task.getPath()) + .setDescription( + Optional.ofNullable(task.getDescription()).orElse("[No description available]")); + if (!log.isEmpty()) { + builder.setLog(toByteArraySupplier(log)); + } + + Reporting> reporting = asReporting(task); + + if (reporting != null) { + // This Task is also a Reporting task! It has a destination file/directory for every supported + // format. + // Add the files for each of the formats into the ReportData. + reporting + .getReports() + .getAsMap() + .forEach( + (type, report) -> { + File destination = report.getDestination(); + // The destination could be a file, or a directory. If it's a directory - the Report + // could have created multiple files - and we need to know to which one of those to + // link. + // + // If we're lucky, whoever implemented the Report made sure to extend + // DirectoryReport, which gives us the entry point to all the files. + // + // This isn't guaranteed though, as it depends on the implementer. + Optional entryPointHint = + destination.isDirectory() && (report instanceof DirectoryReport) + ? Optional.ofNullable(((DirectoryReport) report).getEntryPoint()) + : Optional.empty(); + builder + .reportsBuilder() + .put(type, createReportFiles(destination, entryPointHint, rootDir)); + }); + } + return builder.build(); + } + @TaskAction void uploadResults() { - System.out.format("GcsReportUploader uploading to %s: Unimplemented\n", bucket); - } - - private static Reporting> asReporting(Task task) { - return (Reporting) task; - } - - void addTask(Task task) { - if (!(task instanceof Reporting)) { + System.out.format("GcsReportUploader: bucket= '%s'\n", bucket); + if (isNullOrEmpty(bucket)) { + System.out.format("GcsReportUploader: no bucket defined. Skipping upload\n"); return; } - reportingTasks.add(task); + + try { + uploadResultsToGcs(); + } catch (Throwable e) { + System.out.format("GcsReportUploader: Encountered error %s\n", e); + e.printStackTrace(System.out); + System.out.format("GcsReportUploader: skipping upload\n"); + } + } + + void uploadResultsToGcs() { + checkNotNull(bucket); + ProjectData projectData = createProjectData(); + + Path folder = Paths.get(createUniqueFolderName()); + + StorageOptions.Builder storageOptions = StorageOptions.newBuilder(); + if (!isNullOrEmpty(credentialsFile)) { + try { + storageOptions.setCredentials( + GoogleCredentials.fromStream(new FileInputStream(credentialsFile))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + Storage storage = storageOptions.build().getService(); + + CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData); + ImmutableMap> filesToUpload = coverPageGenerator.getFilesToUpload(); + + System.out.format( + "GcsReportUploader: going to upload %s files to %s/%s\n", + filesToUpload.size(), bucket, folder); + if ("yes".equals(multithreadedUpload)) { + System.out.format("GcsReportUploader: multi-threaded upload\n"); + uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload); + } else { + System.out.format("GcsReportUploader: single threaded upload\n"); + filesToUpload.forEach( + (path, dataSupplier) -> { + System.out.format("GcsReportUploader: Uploading %s\n", path); + uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier); + }); + } + System.out.format( + "GcsReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n", + bucket, folder.resolve(coverPageGenerator.getEntryPoint())); + } + + void setProject(Project project) { + this.project = project; + + for (Project subProject : project.getAllprojects()) { + subProject.getTasks().all(this::addTask); + } + } + + private void addTask(Task task) { + if (task instanceof GcsReportUploader) { + return; + } + tasks.add(task); + StringBuilder log = new StringBuilder(); + checkArgument( + !logs.containsKey(task.getPath()), + "Multiple tasks with the same .getPath()=%s", + task.getPath()); + logs.put(task.getPath(), log); + task.getLogging().addStandardOutputListener(output -> log.append(output)); + task.getLogging().addStandardErrorListener(output -> log.append(output)); task.finalizedBy(this); } + + @SuppressWarnings("unchecked") + private static Reporting> asReporting(Task task) { + if (task instanceof Reporting) { + return (Reporting>) task; + } + return null; + } + + private String createUniqueFolderName() { + return String.format( + "%h-%h-%h-%h", + secureRandom.nextInt(), + secureRandom.nextInt(), + secureRandom.nextInt(), + secureRandom.nextInt()); + } } diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java index 760467323..bb6dda83c 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java @@ -36,8 +36,6 @@ public class GcsReportUploaderPlugin implements Plugin { task.setGroup("uploads"); }); - for (Project subProject : project.getAllprojects()) { - subProject.getTasks().all(reportUploader::addTask); - } + reportUploader.setProject(project); } } diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java new file mode 100644 index 000000000..51c9da76b --- /dev/null +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java @@ -0,0 +1,169 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.gradle.plugin; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import google.registry.gradle.plugin.ProjectData.TaskData; +import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * All the data of a root Gradle project. + * + *

This is basically all the "relevant" data from a Gradle Project, arranged in an immutable and + * more convenient way. + */ +@AutoValue +abstract class ProjectData { + + abstract String name(); + + abstract String description(); + + abstract String gradleVersion(); + + abstract ImmutableMap projectProperties(); + + abstract ImmutableMap systemProperties(); + + abstract ImmutableSet tasksRequested(); + + abstract ImmutableSet tasks(); + + abstract Builder toBuilder(); + + static Builder builder() { + return new AutoValue_ProjectData.Builder(); + } + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setName(String name); + + abstract Builder setDescription(String description); + + abstract Builder setGradleVersion(String gradleVersion); + + abstract Builder setProjectProperties(Map projectProperties); + + abstract Builder setSystemProperties(Map systemProperties); + + abstract Builder setTasksRequested(Iterable tasksRequested); + + abstract ImmutableSet.Builder tasksBuilder(); + + Builder addTask(TaskData task) { + tasksBuilder().add(task); + return this; + } + + abstract ProjectData build(); + } + + /** + * Relevant data to a single Task's. + * + *

Some Tasks are also "Reporting", meaning they create file outputs we want to upload in + * various formats. The format that interests us the most is "html", as that's nicely browsable, + * but they might also have other formats. + */ + @AutoValue + abstract static class TaskData { + + enum State { + /** The task has failed for some reason. */ + FAILURE, + /** The task was actually run and has finished successfully. */ + SUCCESS, + /** The task was up-to-date and successful, and hence didn't need to run again. */ + UP_TO_DATE; + } + + abstract String uniqueName(); + + abstract String description(); + + abstract State state(); + + abstract Optional> log(); + + /** + * Returns the ReportFiles for every report, keyed on the report type. + * + *

The "html" report type is the most interesting, but there are other report formats. + */ + abstract ImmutableMap reports(); + + abstract Builder toBuilder(); + + static Builder builder() { + return new AutoValue_ProjectData_TaskData.Builder(); + } + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setUniqueName(String name); + + abstract Builder setDescription(String description); + + abstract Builder setState(State state); + + abstract Builder setLog(Supplier log); + + abstract ImmutableMap.Builder reportsBuilder(); + + Builder putReport(String type, ReportFiles reportFiles) { + reportsBuilder().put(type, reportFiles); + return this; + } + + abstract TaskData build(); + } + + /** The files for a single format of a specific Task. */ + @AutoValue + abstract static class ReportFiles { + + /** + * All files generated by this report, keyed from their path to a supplier of their content. + * + *

The reason we use a supplier instead of loading the content is in case the content is + * very large... + * + *

Also, no point in doing IO before we need it! + */ + abstract ImmutableMap> files(); + + /** + * The file that gives access (links...) to all the data in the report. + * + *

Guaranteed to be a key in {@link #files} if and only if files isn't empty. + */ + abstract Path entryPoint(); + + static ReportFiles create(ImmutableMap> files, Path entryPoint) { + checkArgument(files.isEmpty() || files.containsKey(entryPoint)); + return new AutoValue_ProjectData_TaskData_ReportFiles(files, entryPoint); + } + } + } +} diff --git a/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css b/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css new file mode 100644 index 000000000..384948449 --- /dev/null +++ b/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css @@ -0,0 +1,27 @@ +body { + font-family: sans-serif; +} +.task_state_SUCCESS { + color: green; +} +.task_state_FAILURE { + color: red; +} +.task_name { + display: block; + font-size: larger; + font-weight: bold; +} +.task_description { + display: block; + margin-left: 1em; + color: gray; +} +.report_links { + margin-left: 1em; +} +.report_link_broken { + text-decoration: line-through; + color: gray; +} + diff --git a/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy b/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy new file mode 100644 index 000000000..2de8c17a1 --- /dev/null +++ b/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy @@ -0,0 +1,107 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{namespace google.registry.gradle.plugin} + +{template .coverPage} + {@param title: string} + {@param cssFiles: list} + {@param projectState: string} + {@param invocation: string} + {@param tasksByState: map]>>} + + {$title} + {for $cssFile in $cssFiles} + + {/for} + +

+

{$title}

+ + Build results for {$invocation} + + + {for $taskState in mapKeys($tasksByState)} + {if length($tasksByState[$taskState]) > 0} + {call .tasksOfState} + {param state: $taskState /} + {param tasks: $tasksByState[$taskState] /} + {/call} + {/if} + {/for} + +
+ +{/template} + +{template .tasksOfState} + {@param state: string} + {@param tasks: list<[uniqueName: string, description: string, log: string, reports: map]>} + +
+

{$state}

+ // Place the tasks with actual reports first, since those are more likely to be useful + {for $task in $tasks} + {if length(mapKeys($task.reports)) > 0} + {call .task} + {param task: $task /} + {/call} + {/if} + {/for} + // Followup with reports without links + {for $task in $tasks} + {if length(mapKeys($task.reports)) == 0} + {call .task} + {param task: $task /} + {/call} + {/if} + {/for} +
+{/template} + +{template .task} + {@param task: [uniqueName: string, description: string, log: string, reports: map]} + {call .taskInternal} + {param uniqueName: $task.uniqueName /} + {param description: $task.description /} + {param log: $task.log /} + {param reports: $task.reports /} + {/call} +{/template} + +{template .taskInternal} + {@param uniqueName: string} + {@param description: string} + {@param log: string} + {@param reports: map} + +
+ {$uniqueName} + {$description} + + {if $log} + [log] + {else} + [log] + {/if} + {for $type in mapKeys($reports)} + {if $reports[$type]} + [{$type}] + {else} + [{$type}] + {/if} + {/for} + +
+{/template} diff --git a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java new file mode 100644 index 000000000..b5f077585 --- /dev/null +++ b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java @@ -0,0 +1,288 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.gradle.plugin; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import google.registry.gradle.plugin.ProjectData.TaskData; +import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.function.Supplier; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link CoverPageGenerator} */ +@RunWith(JUnit4.class) +public final class CoverPageGeneratorTest { + + private static final ProjectData EMPTY_PROJECT = + ProjectData.builder() + .setName("project-name") + .setDescription("project-description") + .setGradleVersion("gradle-version") + .setProjectProperties(ImmutableMap.of("key", "value")) + .setSystemProperties(ImmutableMap.of()) + .setTasksRequested(ImmutableSet.of(":a:task1", ":a:task2")) + .build(); + + private static final TaskData EMPTY_TASK_SUCCESS = + TaskData.builder() + .setUniqueName("task-success") + .setDescription("a successful task") + .setState(TaskData.State.SUCCESS) + .build(); + + private static final TaskData EMPTY_TASK_FAILURE = + TaskData.builder() + .setUniqueName("task-failure") + .setDescription("a failed task") + .setState(TaskData.State.FAILURE) + .build(); + + private static final TaskData EMPTY_TASK_UP_TO_DATE = + TaskData.builder() + .setUniqueName("task-up-to-date") + .setDescription("an up-to-date task") + .setState(TaskData.State.UP_TO_DATE) + .build(); + + private ImmutableMap getGeneratedFiles(ProjectData project) { + CoverPageGenerator coverPageGenerator = new CoverPageGenerator(project); + ImmutableMap> files = coverPageGenerator.getFilesToUpload(); + return files.entrySet().stream() + .collect( + toImmutableMap( + entry -> entry.getKey().toString(), + entry -> new String(entry.getValue().get(), UTF_8))); + } + + private String getContentOfGeneratedFile(ProjectData project, String expectedPath) { + ImmutableMap files = getGeneratedFiles(project); + assertThat(files).containsKey(expectedPath); + return files.get(expectedPath); + } + + private String getCoverPage(ProjectData project) { + return getContentOfGeneratedFile(project, "index.html"); + } + + @Test + public void testGetFilesToUpload_getEntryPoint_isIndexHtml() { + CoverPageGenerator coverPageGenerator = new CoverPageGenerator(EMPTY_PROJECT); + assertThat(coverPageGenerator.getEntryPoint()).isEqualTo(Paths.get("index.html")); + } + + @Test + public void testGetFilesToUpload_containsEntryFile() { + String content = getContentOfGeneratedFile(EMPTY_PROJECT, "index.html"); + assertThat(content) + .contains( + "./gradlew :a:task1 :a:task2 -P key=value"); + } + + @Test + public void testCoverPage_showsFailedTask() { + String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_FAILURE).build()); + assertThat(content).contains("task-failure"); + assertThat(content).contains("

FAILURE

"); + assertThat(content).doesNotContain("

SUCCESS

"); + assertThat(content).doesNotContain("

UP_TO_DATE

"); + } + + @Test + public void testCoverPage_showsSuccessfulTask() { + String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_SUCCESS).build()); + assertThat(content).contains("task-success"); + assertThat(content).doesNotContain("

FAILURE

"); + assertThat(content).contains("

SUCCESS

"); + assertThat(content).doesNotContain("

UP_TO_DATE

"); + } + + @Test + public void testCoverPage_showsUpToDateTask() { + String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_UP_TO_DATE).build()); + assertThat(content).contains("task-up-to-date"); + assertThat(content).doesNotContain("

FAILURE

"); + assertThat(content).doesNotContain("

SUCCESS

"); + assertThat(content).contains("

UP_TO_DATE

"); + } + + @Test + public void testCoverPage_failedAreFirst() { + String content = + getCoverPage( + EMPTY_PROJECT + .toBuilder() + .addTask(EMPTY_TASK_UP_TO_DATE) + .addTask(EMPTY_TASK_FAILURE) + .addTask(EMPTY_TASK_SUCCESS) + .build()); + assertThat(content).contains("

FAILURE

"); + assertThat(content).contains("

SUCCESS

"); + assertThat(content).contains("

UP_TO_DATE

"); + assertThat(content).containsMatch("(?s)

FAILURE

.*

SUCCESS

"); + assertThat(content).containsMatch("(?s)

FAILURE

.*

UP_TO_DATE

"); + assertThat(content).doesNotContainMatch("(?s)

SUCCESS

.*

FAILURE

"); + assertThat(content).doesNotContainMatch("(?s)

UP_TO_DATE

.*

FAILURE

"); + } + + @Test + public void testCoverPage_failingTask_statusIsFailure() { + String content = + getCoverPage( + EMPTY_PROJECT + .toBuilder() + .addTask(EMPTY_TASK_UP_TO_DATE) + .addTask(EMPTY_TASK_FAILURE) + .addTask(EMPTY_TASK_SUCCESS) + .build()); + assertThat(content).contains("Failed: task-failure"); + } + + @Test + public void testCoverPage_noFailingTask_statusIsSuccess() { + String content = + getCoverPage( + EMPTY_PROJECT + .toBuilder() + .addTask(EMPTY_TASK_UP_TO_DATE) + .addTask(EMPTY_TASK_SUCCESS) + .build()); + assertThat(content).contains("Success!"); + } + + @Test + public void testGetFilesToUpload_containsCssFile() { + ImmutableMap files = getGeneratedFiles(EMPTY_PROJECT); + assertThat(files).containsKey("css/style.css"); + assertThat(files.get("css/style.css")).contains("body {"); + assertThat(files.get("index.html")) + .contains(""); + } + + @Test + public void testCreateReportFiles_taskWithLog() { + ImmutableMap files = + getGeneratedFiles( + EMPTY_PROJECT + .toBuilder() + .addTask( + EMPTY_TASK_SUCCESS + .toBuilder() + .setUniqueName("my:name") + .setLog(toByteArraySupplier("my log data")) + .build()) + .build()); + assertThat(files).containsEntry("logs/my:name.log", "my log data"); + assertThat(files.get("index.html")).contains("[log]"); + } + + @Test + public void testCreateReportFiles_taskWithoutLog() { + ImmutableMap files = + getGeneratedFiles( + EMPTY_PROJECT + .toBuilder() + .addTask(EMPTY_TASK_SUCCESS.toBuilder().setUniqueName("my:name").build()) + .build()); + assertThat(files).doesNotContainKey("logs/my:name.log"); + assertThat(files.get("index.html")).contains("[log]"); + } + + @Test + public void testCreateReportFiles_taskWithFilledReport() { + ImmutableMap files = + getGeneratedFiles( + EMPTY_PROJECT + .toBuilder() + .addTask( + EMPTY_TASK_SUCCESS + .toBuilder() + .putReport( + "someReport", + ReportFiles.create( + ImmutableMap.of( + Paths.get("path", "report.txt"), + toByteArraySupplier("report content")), + Paths.get("path", "report.txt"))) + .build()) + .build()); + assertThat(files).containsEntry("path/report.txt", "report content"); + assertThat(files.get("index.html")).contains("[someReport]"); + } + + @Test + public void testCreateReportFiles_taskWithEmptyReport() { + ImmutableMap files = + getGeneratedFiles( + EMPTY_PROJECT + .toBuilder() + .addTask( + EMPTY_TASK_SUCCESS + .toBuilder() + .putReport( + "someReport", + ReportFiles.create(ImmutableMap.of(), Paths.get("path", "report.txt"))) + .build()) + .build()); + assertThat(files).doesNotContainKey("path/report.txt"); + assertThat(files.get("index.html")) + .contains("[someReport]"); + } + + @Test + public void testCreateReportFiles_taskWithLogAndMultipleReports() { + ImmutableMap files = + getGeneratedFiles( + EMPTY_PROJECT + .toBuilder() + .addTask( + EMPTY_TASK_SUCCESS + .toBuilder() + .setUniqueName("my:name") + .setLog(toByteArraySupplier("log data")) + .putReport( + "filledReport", + ReportFiles.create( + ImmutableMap.of( + Paths.get("path-filled", "report.txt"), + toByteArraySupplier("report content"), + Paths.get("path-filled", "other", "file.txt"), + toByteArraySupplier("some other content")), + Paths.get("path-filled", "report.txt"))) + .putReport( + "emptyReport", + ReportFiles.create( + ImmutableMap.of(), Paths.get("path-empty", "report.txt"))) + .build()) + .build()); + assertThat(files).containsEntry("path-filled/report.txt", "report content"); + assertThat(files).containsEntry("path-filled/other/file.txt", "some other content"); + assertThat(files).containsEntry("logs/my:name.log", "log data"); + assertThat(files.get("index.html")) + .contains("[filledReport]"); + assertThat(files.get("index.html")).contains("[log]"); + assertThat(files.get("index.html")) + .contains("[emptyReport]"); + } +} diff --git a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java new file mode 100644 index 000000000..f28434372 --- /dev/null +++ b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java @@ -0,0 +1,291 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.gradle.plugin; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.truth.Truth.assertThat; +import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles; +import static google.registry.gradle.plugin.GcsPluginUtils.getContentType; +import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; +import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath; +import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs; +import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.common.collect.ImmutableMap; +import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link GcsPluginUtilsTest} */ +@RunWith(JUnit4.class) +public final class GcsPluginUtilsTest { + + @Rule public final TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testGetContentType_knownTypes() { + assertThat(getContentType("path/to/file.html")).isEqualTo("text/html"); + assertThat(getContentType("path/to/file.htm")).isEqualTo("text/html"); + assertThat(getContentType("path/to/file.log")).isEqualTo("text/plain"); + assertThat(getContentType("path/to/file.txt")).isEqualTo("text/plain"); + assertThat(getContentType("path/to/file.css")).isEqualTo("text/css"); + assertThat(getContentType("path/to/file.xml")).isEqualTo("text/xml"); + assertThat(getContentType("path/to/file.zip")).isEqualTo("application/zip"); + assertThat(getContentType("path/to/file.js")).isEqualTo("text/javascript"); + } + + @Test + public void testGetContentType_unknownTypes() { + assertThat(getContentType("path/to/file.unknown")).isEqualTo("application/octet-stream"); + } + + @Test + public void testUploadFileToGcs() { + Storage storage = mock(Storage.class); + uploadFileToGcs( + storage, "my-bucket", Paths.get("my", "filename.txt"), toByteArraySupplier("my data")); + verify(storage) + .create( + BlobInfo.newBuilder("my-bucket", "my/filename.txt") + .setContentType("text/plain") + .build(), + "my data".getBytes(UTF_8)); + verifyNoMoreInteractions(storage); + } + + @Test + public void testUploadFilesToGcsMultithread() { + Storage storage = mock(Storage.class); + uploadFilesToGcsMultithread( + storage, + "my-bucket", + Paths.get("my", "folder"), + ImmutableMap.of( + Paths.get("some", "index.html"), toByteArraySupplier("some web page"), + Paths.get("some", "style.css"), toByteArraySupplier("some style"), + Paths.get("other", "index.html"), toByteArraySupplier("other web page"), + Paths.get("other", "style.css"), toByteArraySupplier("other style"))); + verify(storage) + .create( + BlobInfo.newBuilder("my-bucket", "my/folder/some/index.html") + .setContentType("text/html") + .build(), + "some web page".getBytes(UTF_8)); + verify(storage) + .create( + BlobInfo.newBuilder("my-bucket", "my/folder/some/style.css") + .setContentType("text/css") + .build(), + "some style".getBytes(UTF_8)); + verify(storage) + .create( + BlobInfo.newBuilder("my-bucket", "my/folder/other/index.html") + .setContentType("text/html") + .build(), + "other web page".getBytes(UTF_8)); + verify(storage) + .create( + BlobInfo.newBuilder("my-bucket", "my/folder/other/style.css") + .setContentType("text/css") + .build(), + "other style".getBytes(UTF_8)); + verifyNoMoreInteractions(storage); + } + + @Test + public void testToByteArraySupplier_string() { + assertThat(toByteArraySupplier("my string").get()).isEqualTo("my string".getBytes(UTF_8)); + } + + @Test + public void testToByteArraySupplier_stringSupplier() { + assertThat(toByteArraySupplier(() -> "my string").get()).isEqualTo("my string".getBytes(UTF_8)); + } + + @Test + public void testToByteArraySupplier_file() throws Exception { + folder.newFolder("arbitrary"); + File file = folder.newFile("arbitrary/file.txt"); + Files.write(file.toPath(), "some data".getBytes(UTF_8)); + assertThat(toByteArraySupplier(file).get()).isEqualTo("some data".getBytes(UTF_8)); + } + + private ImmutableMap readAllFiles(ReportFiles reportFiles) { + return reportFiles.files().entrySet().stream() + .collect( + toImmutableMap( + entry -> entry.getKey().toString(), + entry -> new String(entry.getValue().get(), UTF_8))); + } + + @Test + public void testCreateReportFiles_destinationIsFile() throws Exception { + Path root = toNormalizedPath(folder.newFolder("my", "root")); + folder.newFolder("my", "root", "some", "path"); + File destination = folder.newFile("my/root/some/path/file.txt"); + Files.write(destination.toPath(), "some data".getBytes(UTF_8)); + // Since the entry point is obvious here - any hint given is just ignored. + File ignoredHint = folder.newFile("my/root/ignored.txt"); + + ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + + assertThat(files.entryPoint().toString()).isEqualTo("some/path/file.txt"); + assertThat(readAllFiles(files)).containsExactly("some/path/file.txt", "some data"); + } + + @Test + public void testCreateReportFiles_destinationDoesntExist() throws Exception { + Path root = toNormalizedPath(folder.newFolder("my", "root")); + File destination = root.resolve("non/existing.txt").toFile(); + assertThat(destination.isFile()).isFalse(); + assertThat(destination.isDirectory()).isFalse(); + // Since there are not files, any hint given is obvioulsy wrong and will be ignored. + File ignoredHint = folder.newFile("my/root/ignored.txt"); + + ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + + assertThat(files.entryPoint().toString()).isEqualTo("non/existing.txt"); + assertThat(files.files()).isEmpty(); + } + + @Test + public void testCreateReportFiles_noFiles() throws Exception { + Path root = toNormalizedPath(folder.newFolder("my", "root")); + File destination = folder.newFolder("my", "root", "some", "path"); + folder.newFolder("my", "root", "some", "path", "a", "b"); + folder.newFolder("my", "root", "some", "path", "c"); + // Since there are not files, any hint given is obvioulsy wrong and will be ignored. + File ignoredHint = folder.newFile("my/root/ignored.txt"); + + ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + + assertThat(files.entryPoint().toString()).isEqualTo("some/path"); + assertThat(files.files()).isEmpty(); + } + + @Test + public void testCreateReportFiles_oneFile() throws Exception { + Path root = toNormalizedPath(folder.newFolder("my", "root")); + File destination = folder.newFolder("my", "root", "some", "path"); + folder.newFolder("my", "root", "some", "path", "a", "b"); + folder.newFolder("my", "root", "some", "path", "c"); + Files.write( + folder.newFile("my/root/some/path/a/file.txt").toPath(), "some data".getBytes(UTF_8)); + // Since the entry point is obvious here - any hint given is just ignored. + File ignoredHint = folder.newFile("my/root/ignored.txt"); + + ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + + assertThat(files.entryPoint().toString()).isEqualTo("some/path/a/file.txt"); + assertThat(readAllFiles(files)).containsExactly("some/path/a/file.txt", "some data"); + } + + /** + * Currently tests the "unimplemented" behavior. + * + *

TODO(guyben): switch to checking zip file instead. + */ + @Test + public void testCreateReportFiles_multipleFiles_noHint() throws Exception { + Path root = toNormalizedPath(folder.newFolder("my", "root")); + File destination = folder.newFolder("my", "root", "some", "path"); + folder.newFolder("my", "root", "some", "path", "a", "b"); + folder.newFolder("my", "root", "some", "path", "c"); + + Files.write( + folder.newFile("my/root/some/path/index.html").toPath(), "some data".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); + + ReportFiles files = createReportFiles(destination, Optional.empty(), root); + + assertThat(files.entryPoint().toString()).isEqualTo("some/path/unimplemented.txt"); + assertThat(readAllFiles(files).keySet()).containsExactly("some/path/unimplemented.txt"); + } + + /** + * Currently tests the "unimplemented" behavior. + * + *

TODO(guyben): switch to checking zip file instead. + */ + @Test + public void testCreateReportFiles_multipleFiles_withBadHint() throws Exception { + Path root = toNormalizedPath(folder.newFolder("my", "root")); + File destination = folder.newFolder("my", "root", "some", "path"); + // This entry point points to a directory, which isn't an appropriate entry point + File badEntryPoint = folder.newFolder("my", "root", "some", "path", "a", "b"); + folder.newFolder("my", "root", "some", "path", "c"); + + Files.write( + folder.newFile("my/root/some/path/index.html").toPath(), "some data".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); + + ReportFiles files = createReportFiles(destination, Optional.of(badEntryPoint), root); + + assertThat(files.entryPoint().toString()).isEqualTo("some/path/unimplemented.txt"); + assertThat(readAllFiles(files).keySet()).containsExactly("some/path/unimplemented.txt"); + } + + @Test + public void testCreateReportFiles_multipleFiles_withGoodHint() throws Exception { + Path root = toNormalizedPath(folder.newFolder("my", "root")); + File destination = folder.newFolder("my", "root", "some", "path"); + folder.newFolder("my", "root", "some", "path", "a", "b"); + folder.newFolder("my", "root", "some", "path", "c"); + // The hint is an actual file nested in the destination directory! + File goodEntryPoint = folder.newFile("my/root/some/path/index.html"); + + Files.write(goodEntryPoint.toPath(), "some data".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8)); + Files.write( + folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); + + ReportFiles files = createReportFiles(destination, Optional.of(goodEntryPoint), root); + + assertThat(files.entryPoint().toString()).isEqualTo("some/path/index.html"); + assertThat(readAllFiles(files)) + .containsExactly( + "some/path/index.html", "some data", + "some/path/a/index.html", "wrong index", + "some/path/c/style.css", "css file", + "some/path/my_image.png", "images"); + } +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 2246d96b6..9bacfac1e 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -36,6 +36,7 @@ ext { 'com.google.auto.value:auto-value:1.6.2', 'com.google.auto.value:auto-value-annotations:1.6.2', 'com.google.closure-stylesheets:closure-stylesheets:1.5.0', + 'com.google.cloud:google-cloud-storage:1.59.0', 'com.google.cloud.bigdataoss:gcsio:1.4.5', 'com.google.cloud.bigdataoss:util:1.4.5', 'com.googlecode.charts4j:charts4j:1.3', diff --git a/gradle/gradle.properties b/gradle/gradle.properties index 8035c0843..ded949e07 100644 --- a/gradle/gradle.properties +++ b/gradle/gradle.properties @@ -1,2 +1,5 @@ repositoryUrl= publishUrl= +gcsBucket= +gcsCredentialsFile= +gcsMultithreadedUpload=