diff --git a/gradle/build.gradle b/gradle/build.gradle
index d540471d2..6313237a2 100644
--- a/gradle/build.gradle
+++ b/gradle/build.gradle
@@ -26,7 +26,24 @@ plugins {
apply plugin: google.registry.gradle.plugin.GcsReportUploaderPlugin
gcsReportUploader {
- bucket = 'my-bucket'
+ // Set the bucket here to upload build results to a GCS bucket.
+ // e.g. -P gcsBucket=domain-registry-alpha-build-result-test
+ //
+ // If no bucket it set - the uploading will be skipped.
+ bucket = gcsBucket
+
+ // The location of the file containing the OAuth2 Google Cloud credentials.
+ //
+ // The file can contain a Service Account key file in JSON format from the
+ // Google Developers Console or a stored user credential using the format
+ // supported by the Cloud SDK.
+ //
+ // If no file is given - the default credentials are used.
+ credentialsFile = gcsCredentialsFile
+
+ // If set to 'yes', each file will be uploaded to GCS in a separate thread.
+ // This is MUCH faster.
+ multithreadedUpload = gcsMultithreadedUpload
}
apply from: 'dependencies.gradle'
diff --git a/gradle/buildSrc/build.gradle b/gradle/buildSrc/build.gradle
index e27d22cab..bf8c78c3f 100644
--- a/gradle/buildSrc/build.gradle
+++ b/gradle/buildSrc/build.gradle
@@ -17,5 +17,19 @@ apply from: '../dependencies.gradle'
dependencies {
def deps = dependencyMap
compile deps['com.google.guava:guava']
+ compile deps['com.google.auto.value:auto-value-annotations']
+ compile deps['com.google.cloud:google-cloud-storage']
+ compile deps['org.apache.commons:commons-text']
+ compile deps['com.google.auth:google-auth-library-credentials']
+ compile deps['com.google.template:soy']
+ annotationProcessor deps['com.google.auto.value:auto-value']
testCompile deps['com.google.truth:truth']
+ testCompile deps['com.google.truth.extensions:truth-java8-extension']
+ testCompile deps['org.mockito:mockito-all']
+}
+
+gradle.projectsEvaluated {
+ tasks.withType(JavaCompile) {
+ options.compilerArgs << "-Xlint:unchecked"
+ }
}
diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java
new file mode 100644
index 000000000..f71c60607
--- /dev/null
+++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java
@@ -0,0 +1,207 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.gradle.plugin;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
+import static com.google.common.io.Resources.getResource;
+import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.template.soy.SoyFileSet;
+import com.google.template.soy.tofu.SoyTofu;
+import google.registry.gradle.plugin.ProjectData.TaskData;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+/**
+ * Creates the files for a web-page summary of a given {@Link ProjectData}.
+ *
+ *
The main job of this class is rendering a tailored cover page that includes information about
+ * the project and any task that ran.
+ *
+ *
It returns all the files that need uploading for the cover page to work. This includes any
+ * report and log files linked to in the ProjectData, as well as a cover page (and associated
+ * resources such as CSS files).
+ */
+final class CoverPageGenerator {
+
+ /** List of all resource files that will be uploaded as-is. */
+ private static final ImmutableSet STATIC_RESOURCE_FILES =
+ ImmutableSet.of(Paths.get("css", "style.css"));
+
+ private final ProjectData projectData;
+ private final ImmutableSetMultimap tasksByState;
+ /**
+ * The compiled SOY files.
+ *
+ *
Will be generated only when actually needed, because it takes a while to compile and we
+ * don't want that to happen unless we actually use it.
+ */
+ private SoyTofu tofu = null;
+
+ CoverPageGenerator(ProjectData projectData) {
+ this.projectData = projectData;
+ this.tasksByState =
+ projectData.tasks().stream().collect(toImmutableSetMultimap(TaskData::state, task -> task));
+ }
+
+ /**
+ * The (relative) entry point for the cover page.
+ *
+ *
A file with this relative path is guaranteed to be returned from {@link #getFilesToUpload},
+ * and a browser pointing to this page will have access to all the data generated by {@link
+ * #getFilesToUpload}.
+ */
+ Path getEntryPoint() {
+ return Paths.get("index.html");
+ }
+
+ /**
+ * Returns all the files that need uploading for the cover page to work.
+ *
+ *
This includes all the report files as well, to make sure that the link works.
+ */
+ ImmutableMap> getFilesToUpload() {
+ ImmutableMap.Builder> builder = new ImmutableMap.Builder<>();
+ // Add all the static resource pages
+ STATIC_RESOURCE_FILES.stream().forEach(file -> builder.put(file, resourceLoader(file)));
+ // Create the cover page
+ // Note that the ByteArraySupplier here is lazy - the createCoverPage function is only called
+ // when the resulting Supplier's get function is called.
+ builder.put(getEntryPoint(), toByteArraySupplier(this::createCoverPage));
+ // Add all the files from the tasks
+ tasksByState.values().stream()
+ .flatMap(task -> task.reports().values().stream())
+ .forEach(reportFiles -> builder.putAll(reportFiles.files()));
+ // Add the logs of every test
+ tasksByState.values().stream()
+ .filter(task -> task.log().isPresent())
+ .forEach(task -> builder.put(getLogPath(task), task.log().get()));
+
+ return builder.build();
+ }
+
+ /** Renders the cover page. */
+ private String createCoverPage() {
+ return getTofu()
+ .newRenderer("google.registry.gradle.plugin.coverPage")
+ .setData(getSoyData())
+ .render();
+ }
+
+ /** Converts the projectData and all taskData into all the data the soy template needs. */
+ private ImmutableMap getSoyData() {
+ ImmutableMap.Builder builder = new ImmutableMap.Builder<>();
+
+ TaskData.State state =
+ tasksByState.containsKey(TaskData.State.FAILURE)
+ ? TaskData.State.FAILURE
+ : TaskData.State.SUCCESS;
+ String title =
+ state != TaskData.State.FAILURE
+ ? "Success!"
+ : "Failed: "
+ + tasksByState.get(state).stream()
+ .map(TaskData::uniqueName)
+ .collect(Collectors.joining(", "));
+
+ builder.put("projectState", state.toString());
+ builder.put("title", title);
+ builder.put("cssFiles", ImmutableSet.of("css/style.css"));
+ builder.put("invocation", getInvocation());
+ builder.put("tasksByState", getTasksByStateSoyData());
+ return builder.build();
+ }
+
+ /**
+ * Returns a soy-friendly map from the TaskData.State to the task itslef.
+ *
+ *
The key order in the resulting map is always the same (the order from the enum definition)
+ * no matter the key order in the original tasksByState map.
+ */
+ private ImmutableMap getTasksByStateSoyData() {
+ ImmutableMap.Builder builder = new ImmutableMap.Builder<>();
+
+ // We go over the States in the order they are defined rather than the order in which they
+ // happen to be in the tasksByState Map.
+ //
+ // That way we guarantee a consistent order.
+ for (TaskData.State state : TaskData.State.values()) {
+ builder.put(
+ state.toString(),
+ tasksByState.get(state).stream()
+ .map(task -> taskDataToSoy(task))
+ .collect(toImmutableList()));
+ }
+
+ return builder.build();
+ }
+
+ /** returns a soy-friendly version of the given task data. */
+ static ImmutableMap taskDataToSoy(TaskData task) {
+ return new ImmutableMap.Builder()
+ .put("uniqueName", task.uniqueName())
+ .put("description", task.description())
+ .put("log", task.log().isPresent() ? getLogPath(task).toString() : "")
+ .put(
+ "reports",
+ task.reports().entrySet().stream()
+ .collect(
+ toImmutableMap(
+ entry -> entry.getKey(),
+ entry ->
+ entry.getValue().files().isEmpty()
+ ? ""
+ : entry.getValue().entryPoint().toString())))
+ .build();
+ }
+
+ private String getInvocation() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("./gradlew");
+ projectData.tasksRequested().forEach(task -> builder.append(" ").append(task));
+ projectData
+ .projectProperties()
+ .forEach((key, value) -> builder.append(String.format(" -P %s=%s", key, value)));
+ return builder.toString();
+ }
+
+ /** Returns a lazily created soy renderer */
+ private SoyTofu getTofu() {
+ if (tofu == null) {
+ tofu =
+ SoyFileSet.builder()
+ .add(getResource(CoverPageGenerator.class, "soy/coverpage.soy"))
+ .build()
+ .compileToTofu();
+ }
+ return tofu;
+ }
+
+ private static Path getLogPath(TaskData task) {
+ // TODO(guyben):escape uniqueName to guarantee legal file name.
+ return Paths.get("logs", task.uniqueName() + ".log");
+ }
+
+ private static Supplier resourceLoader(Path path) {
+ return toByteArraySupplier(getResource(CoverPageGenerator.class, path.toString()));
+ }
+}
diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java
new file mode 100644
index 000000000..489c91f0c
--- /dev/null
+++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java
@@ -0,0 +1,206 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.gradle.plugin;
+
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static com.google.common.collect.Iterables.getOnlyElement;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
+import com.google.common.io.Files;
+import com.google.common.io.Resources;
+import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+/** Utility functions used in the GCS plugin. */
+final class GcsPluginUtils {
+
+ private static final ImmutableMap EXTENSION_TO_CONTENT_TYPE =
+ new ImmutableMap.Builder()
+ .put("html", "text/html")
+ .put("htm", "text/html")
+ .put("log", "text/plain")
+ .put("txt", "text/plain")
+ .put("css", "text/css")
+ .put("xml", "text/xml")
+ .put("zip", "application/zip")
+ .put("js", "text/javascript")
+ .build();
+
+ private static final String DEFAULT_CONTENT_TYPE = "application/octet-stream";
+
+ static Path toNormalizedPath(File file) {
+ return file.toPath().toAbsolutePath().normalize();
+ }
+
+ static String getContentType(String fileName) {
+ return EXTENSION_TO_CONTENT_TYPE.getOrDefault(
+ Files.getFileExtension(fileName), DEFAULT_CONTENT_TYPE);
+ }
+
+ static void uploadFileToGcs(
+ Storage storage, String bucket, Path path, Supplier dataSupplier) {
+ String filename = path.toString();
+ storage.create(
+ BlobInfo.newBuilder(bucket, filename).setContentType(getContentType(filename)).build(),
+ dataSupplier.get());
+ }
+
+ static void uploadFilesToGcsMultithread(
+ Storage storage, String bucket, Path folder, Map> files) {
+ ImmutableMap.Builder threads = new ImmutableMap.Builder<>();
+ files.forEach(
+ (path, dataSupplier) -> {
+ Thread thread =
+ new Thread(
+ () -> uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier));
+ thread.start();
+ threads.put(path, thread);
+ });
+ threads
+ .build()
+ .forEach(
+ (path, thread) -> {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ System.out.format("Upload of %s interrupted", path);
+ }
+ });
+ }
+
+ static Supplier toByteArraySupplier(String data) {
+ return () -> data.getBytes(UTF_8);
+ }
+
+ static Supplier toByteArraySupplier(Supplier dataSupplier) {
+ return () -> dataSupplier.get().getBytes(UTF_8);
+ }
+
+ static Supplier toByteArraySupplier(File file) {
+ return () -> {
+ try {
+ return Files.toByteArray(file);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ };
+ }
+
+ static Supplier toByteArraySupplier(URL url) {
+ return () -> {
+ try {
+ return Resources.toByteArray(url);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ };
+ }
+
+ /**
+ * Reads all the files generated by a Report into a ReportFiles object.
+ *
+ *
Every ReportFiles must have a single link "entry point" that gives users access to all the
+ * files. If the report generated just one file - we will just link to that file.
+ *
+ *
However, if the report generated more than one file - the only thing we can safely do is to
+ * zip all the files and link to the zip file.
+ *
+ *
As an alternative to using a zip file, we allow the caller to supply an optional "entry
+ * point" file that will link to all the other files. If that file is given and is "appropriate"
+ * (exists and is in the correct location) - we will upload all the report files "as is" and link
+ * to the entry file.
+ *
+ * @param destination the location of the output. Either a file or a directory. If a directory -
+ * then all the files inside that directory are the outputs we're looking for.
+ * @param entryPointHint If present - a hint to what the entry point to this directory tree is.
+ * Will only be used if all of the following apply: (a) {@code
+ * destination.isDirectory()==true}, (b) there are 2 or more files in the {@code destination}
+ * directory, and (c) {@code entryPointHint.get()} is one of the files nested inside of the
+ * {@code destination} directory.
+ */
+ static ReportFiles createReportFiles(
+ File destination, Optional entryPointHint, Path rootDir) {
+
+ Path destinationPath = rootDir.relativize(toNormalizedPath(destination));
+
+ if (destination.isFile()) {
+ // The destination is a single file - find its root, and add this single file to the
+ // ReportFiles.
+ return ReportFiles.create(
+ ImmutableMap.of(destinationPath, toByteArraySupplier(destination)), destinationPath);
+ }
+
+ if (!destination.isDirectory()) {
+ // This isn't a file nor a directory - so it doesn't exist! Return empty ReportFiles
+ return ReportFiles.create(ImmutableMap.of(), destinationPath);
+ }
+
+ // The destination is a directory - find all the actual files first
+ ImmutableMap> files =
+ Streams.stream(Files.fileTraverser().depthFirstPreOrder(destination))
+ .filter(File::isFile)
+ .collect(
+ toImmutableMap(
+ file -> rootDir.relativize(toNormalizedPath(file)),
+ file -> toByteArraySupplier(file)));
+
+ if (files.isEmpty()) {
+ // The directory exists, but is empty. Return empty ReportFiles
+ return ReportFiles.create(ImmutableMap.of(), destinationPath);
+ }
+
+ if (files.size() == 1) {
+ // We got a directory, but it only has a single file. We can link to that.
+ return ReportFiles.create(files, getOnlyElement(files.keySet()));
+ }
+
+ // There are multiple files in the report! We need to check the entryPointHint
+ Optional entryPointPath =
+ entryPointHint.map(file -> rootDir.relativize(toNormalizedPath(file)));
+
+ if (entryPointPath.isPresent() && files.containsKey(entryPointPath.get())) {
+ // We were given the entry point! Use it!
+ return ReportFiles.create(files, entryPointPath.get());
+ }
+
+ // We weren't given an appropriate entry point. But we still need a single link to all this data
+ // - so we'll zip it and just host a single file.
+ //
+ // TODO(guyben):the zip part is still unimplemented, but what we'll want to do is this:
+ // Supplier zippedSupplier = createZippedByteArraySupplier(files);
+ // Path zipFilePath = rootFolder.resolve(rootFolder.getFileName().toString() + ".zip");
+ // return ReportFiles.create(ImmutableMap.of(zipFilePath, zippedSupplier), zipFilePath);
+ Path unimplementedPath = destinationPath.resolve("unimplemented.txt");
+ String content =
+ "Zip files are currently unimplemented. Files:\n"
+ + files.keySet().stream().map(Object::toString).collect(Collectors.joining("\n"));
+ return ReportFiles.create(
+ ImmutableMap.of(unimplementedPath, toByteArraySupplier(content)), unimplementedPath);
+ }
+
+ private GcsPluginUtils() {}
+}
diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java
index 04f3f287f..4d06eaf5a 100644
--- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java
+++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java
@@ -14,9 +14,35 @@
package google.registry.gradle.plugin;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles;
+import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
+import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath;
+import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs;
+import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread;
+
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import com.google.common.collect.ImmutableMap;
+import google.registry.gradle.plugin.ProjectData.TaskData;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.SecureRandom;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Optional;
+import java.util.function.Supplier;
import org.gradle.api.DefaultTask;
+import org.gradle.api.Project;
import org.gradle.api.Task;
+import org.gradle.api.reporting.DirectoryReport;
import org.gradle.api.reporting.Report;
import org.gradle.api.reporting.ReportContainer;
import org.gradle.api.reporting.Reporting;
@@ -24,33 +50,200 @@ import org.gradle.api.tasks.TaskAction;
/**
* A task that uploads the Reports generated by other tasks to GCS.
- *
- *
TODO:implement it.
*/
public class GcsReportUploader extends DefaultTask {
- private final ArrayList reportingTasks = new ArrayList<>();
+ private static final SecureRandom secureRandom = new SecureRandom();
+
+ private final ArrayList tasks = new ArrayList<>();
+ private final HashMap logs = new HashMap<>();
+ private Project project;
private String bucket = null;
+ private String credentialsFile = null;
+ private String multithreadedUpload = null;
public void setBucket(String bucket) {
this.bucket = bucket;
}
+ public void setCredentialsFile(String credentialsFile) {
+ this.credentialsFile = credentialsFile;
+ }
+
+ public void setMultithreadedUpload(String multithreadedUpload) {
+ this.multithreadedUpload = multithreadedUpload;
+ }
+
+ /** Converts the given Gradle Project into a ProjectData. */
+ private ProjectData createProjectData() {
+ ProjectData.Builder builder =
+ ProjectData.builder()
+ .setName(project.getPath() + project.getName())
+ .setDescription(
+ Optional.ofNullable(project.getDescription()).orElse("[No description available]"))
+ .setGradleVersion(project.getGradle().getGradleVersion())
+ .setProjectProperties(project.getGradle().getStartParameter().getProjectProperties())
+ .setSystemProperties(project.getGradle().getStartParameter().getSystemPropertiesArgs())
+ .setTasksRequested(project.getGradle().getStartParameter().getTaskNames());
+
+ Path rootDir = toNormalizedPath(project.getRootDir());
+ tasks.stream()
+ .filter(task -> task.getState().getExecuted() || task.getState().getUpToDate())
+ .map(task -> createTaskData(task, rootDir))
+ .forEach(builder.tasksBuilder()::add);
+ return builder.build();
+ }
+
+ /**
+ * Converts a Gradle Task into a TaskData.
+ *
+ * @param rootDir the root directory of the main Project - used to get the relative path of any
+ * Task files.
+ */
+ private TaskData createTaskData(Task task, Path rootDir) {
+ TaskData.State state =
+ task.getState().getFailure() != null
+ ? TaskData.State.FAILURE
+ : task.getState().getUpToDate() ? TaskData.State.UP_TO_DATE : TaskData.State.SUCCESS;
+ String log = logs.get(task.getPath()).toString();
+
+ TaskData.Builder builder =
+ TaskData.builder()
+ .setState(state)
+ .setUniqueName(task.getPath())
+ .setDescription(
+ Optional.ofNullable(task.getDescription()).orElse("[No description available]"));
+ if (!log.isEmpty()) {
+ builder.setLog(toByteArraySupplier(log));
+ }
+
+ Reporting> reporting = asReporting(task);
+
+ if (reporting != null) {
+ // This Task is also a Reporting task! It has a destination file/directory for every supported
+ // format.
+ // Add the files for each of the formats into the ReportData.
+ reporting
+ .getReports()
+ .getAsMap()
+ .forEach(
+ (type, report) -> {
+ File destination = report.getDestination();
+ // The destination could be a file, or a directory. If it's a directory - the Report
+ // could have created multiple files - and we need to know to which one of those to
+ // link.
+ //
+ // If we're lucky, whoever implemented the Report made sure to extend
+ // DirectoryReport, which gives us the entry point to all the files.
+ //
+ // This isn't guaranteed though, as it depends on the implementer.
+ Optional entryPointHint =
+ destination.isDirectory() && (report instanceof DirectoryReport)
+ ? Optional.ofNullable(((DirectoryReport) report).getEntryPoint())
+ : Optional.empty();
+ builder
+ .reportsBuilder()
+ .put(type, createReportFiles(destination, entryPointHint, rootDir));
+ });
+ }
+ return builder.build();
+ }
+
@TaskAction
void uploadResults() {
- System.out.format("GcsReportUploader uploading to %s: Unimplemented\n", bucket);
- }
-
- private static Reporting> asReporting(Task task) {
- return (Reporting) task;
- }
-
- void addTask(Task task) {
- if (!(task instanceof Reporting)) {
+ System.out.format("GcsReportUploader: bucket= '%s'\n", bucket);
+ if (isNullOrEmpty(bucket)) {
+ System.out.format("GcsReportUploader: no bucket defined. Skipping upload\n");
return;
}
- reportingTasks.add(task);
+
+ try {
+ uploadResultsToGcs();
+ } catch (Throwable e) {
+ System.out.format("GcsReportUploader: Encountered error %s\n", e);
+ e.printStackTrace(System.out);
+ System.out.format("GcsReportUploader: skipping upload\n");
+ }
+ }
+
+ void uploadResultsToGcs() {
+ checkNotNull(bucket);
+ ProjectData projectData = createProjectData();
+
+ Path folder = Paths.get(createUniqueFolderName());
+
+ StorageOptions.Builder storageOptions = StorageOptions.newBuilder();
+ if (!isNullOrEmpty(credentialsFile)) {
+ try {
+ storageOptions.setCredentials(
+ GoogleCredentials.fromStream(new FileInputStream(credentialsFile)));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ Storage storage = storageOptions.build().getService();
+
+ CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData);
+ ImmutableMap> filesToUpload = coverPageGenerator.getFilesToUpload();
+
+ System.out.format(
+ "GcsReportUploader: going to upload %s files to %s/%s\n",
+ filesToUpload.size(), bucket, folder);
+ if ("yes".equals(multithreadedUpload)) {
+ System.out.format("GcsReportUploader: multi-threaded upload\n");
+ uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload);
+ } else {
+ System.out.format("GcsReportUploader: single threaded upload\n");
+ filesToUpload.forEach(
+ (path, dataSupplier) -> {
+ System.out.format("GcsReportUploader: Uploading %s\n", path);
+ uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier);
+ });
+ }
+ System.out.format(
+ "GcsReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n",
+ bucket, folder.resolve(coverPageGenerator.getEntryPoint()));
+ }
+
+ void setProject(Project project) {
+ this.project = project;
+
+ for (Project subProject : project.getAllprojects()) {
+ subProject.getTasks().all(this::addTask);
+ }
+ }
+
+ private void addTask(Task task) {
+ if (task instanceof GcsReportUploader) {
+ return;
+ }
+ tasks.add(task);
+ StringBuilder log = new StringBuilder();
+ checkArgument(
+ !logs.containsKey(task.getPath()),
+ "Multiple tasks with the same .getPath()=%s",
+ task.getPath());
+ logs.put(task.getPath(), log);
+ task.getLogging().addStandardOutputListener(output -> log.append(output));
+ task.getLogging().addStandardErrorListener(output -> log.append(output));
task.finalizedBy(this);
}
+
+ @SuppressWarnings("unchecked")
+ private static Reporting> asReporting(Task task) {
+ if (task instanceof Reporting) {
+ return (Reporting>) task;
+ }
+ return null;
+ }
+
+ private String createUniqueFolderName() {
+ return String.format(
+ "%h-%h-%h-%h",
+ secureRandom.nextInt(),
+ secureRandom.nextInt(),
+ secureRandom.nextInt(),
+ secureRandom.nextInt());
+ }
}
diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java
index 760467323..bb6dda83c 100644
--- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java
+++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java
@@ -36,8 +36,6 @@ public class GcsReportUploaderPlugin implements Plugin {
task.setGroup("uploads");
});
- for (Project subProject : project.getAllprojects()) {
- subProject.getTasks().all(reportUploader::addTask);
- }
+ reportUploader.setProject(project);
}
}
diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java
new file mode 100644
index 000000000..51c9da76b
--- /dev/null
+++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java
@@ -0,0 +1,169 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.gradle.plugin;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.gradle.plugin.ProjectData.TaskData;
+import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+/**
+ * All the data of a root Gradle project.
+ *
+ *
This is basically all the "relevant" data from a Gradle Project, arranged in an immutable and
+ * more convenient way.
+ */
+@AutoValue
+abstract class ProjectData {
+
+ abstract String name();
+
+ abstract String description();
+
+ abstract String gradleVersion();
+
+ abstract ImmutableMap projectProperties();
+
+ abstract ImmutableMap systemProperties();
+
+ abstract ImmutableSet tasksRequested();
+
+ abstract ImmutableSet tasks();
+
+ abstract Builder toBuilder();
+
+ static Builder builder() {
+ return new AutoValue_ProjectData.Builder();
+ }
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setName(String name);
+
+ abstract Builder setDescription(String description);
+
+ abstract Builder setGradleVersion(String gradleVersion);
+
+ abstract Builder setProjectProperties(Map projectProperties);
+
+ abstract Builder setSystemProperties(Map systemProperties);
+
+ abstract Builder setTasksRequested(Iterable tasksRequested);
+
+ abstract ImmutableSet.Builder tasksBuilder();
+
+ Builder addTask(TaskData task) {
+ tasksBuilder().add(task);
+ return this;
+ }
+
+ abstract ProjectData build();
+ }
+
+ /**
+ * Relevant data to a single Task's.
+ *
+ *
Some Tasks are also "Reporting", meaning they create file outputs we want to upload in
+ * various formats. The format that interests us the most is "html", as that's nicely browsable,
+ * but they might also have other formats.
+ */
+ @AutoValue
+ abstract static class TaskData {
+
+ enum State {
+ /** The task has failed for some reason. */
+ FAILURE,
+ /** The task was actually run and has finished successfully. */
+ SUCCESS,
+ /** The task was up-to-date and successful, and hence didn't need to run again. */
+ UP_TO_DATE;
+ }
+
+ abstract String uniqueName();
+
+ abstract String description();
+
+ abstract State state();
+
+ abstract Optional> log();
+
+ /**
+ * Returns the ReportFiles for every report, keyed on the report type.
+ *
+ *
The "html" report type is the most interesting, but there are other report formats.
+ */
+ abstract ImmutableMap reports();
+
+ abstract Builder toBuilder();
+
+ static Builder builder() {
+ return new AutoValue_ProjectData_TaskData.Builder();
+ }
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setUniqueName(String name);
+
+ abstract Builder setDescription(String description);
+
+ abstract Builder setState(State state);
+
+ abstract Builder setLog(Supplier log);
+
+ abstract ImmutableMap.Builder reportsBuilder();
+
+ Builder putReport(String type, ReportFiles reportFiles) {
+ reportsBuilder().put(type, reportFiles);
+ return this;
+ }
+
+ abstract TaskData build();
+ }
+
+ /** The files for a single format of a specific Task. */
+ @AutoValue
+ abstract static class ReportFiles {
+
+ /**
+ * All files generated by this report, keyed from their path to a supplier of their content.
+ *
+ *
The reason we use a supplier instead of loading the content is in case the content is
+ * very large...
+ *
+ *
Also, no point in doing IO before we need it!
+ */
+ abstract ImmutableMap> files();
+
+ /**
+ * The file that gives access (links...) to all the data in the report.
+ *
+ *
Guaranteed to be a key in {@link #files} if and only if files isn't empty.
+ */
+ abstract Path entryPoint();
+
+ static ReportFiles create(ImmutableMap> files, Path entryPoint) {
+ checkArgument(files.isEmpty() || files.containsKey(entryPoint));
+ return new AutoValue_ProjectData_TaskData_ReportFiles(files, entryPoint);
+ }
+ }
+ }
+}
diff --git a/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css b/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css
new file mode 100644
index 000000000..384948449
--- /dev/null
+++ b/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css
@@ -0,0 +1,27 @@
+body {
+ font-family: sans-serif;
+}
+.task_state_SUCCESS {
+ color: green;
+}
+.task_state_FAILURE {
+ color: red;
+}
+.task_name {
+ display: block;
+ font-size: larger;
+ font-weight: bold;
+}
+.task_description {
+ display: block;
+ margin-left: 1em;
+ color: gray;
+}
+.report_links {
+ margin-left: 1em;
+}
+.report_link_broken {
+ text-decoration: line-through;
+ color: gray;
+}
+
diff --git a/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy b/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy
new file mode 100644
index 000000000..2de8c17a1
--- /dev/null
+++ b/gradle/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy
@@ -0,0 +1,107 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{namespace google.registry.gradle.plugin}
+
+{template .coverPage}
+ {@param title: string}
+ {@param cssFiles: list}
+ {@param projectState: string}
+ {@param invocation: string}
+ {@param tasksByState: map]>>}
+
+ {$title}
+ {for $cssFile in $cssFiles}
+
+ {/for}
+
+