diff --git a/build.gradle b/build.gradle index c931c878a..b36e31418 100644 --- a/build.gradle +++ b/build.gradle @@ -36,8 +36,7 @@ buildscript { } plugins { - // Java static analysis plugins. Keep versions consistent with - // ./buildSrc/build.gradle.kts + // Java static analysis plugins. // Re-enable when compatible with Gradle 8 // id 'nebula.lint' version '16.0.2' @@ -68,44 +67,12 @@ wrapper { distributionType = Wrapper.DistributionType.ALL } -apply plugin: google.registry.gradle.plugin.ReportUploaderPlugin - -reportUploader { - // Set the location where we want to upload the build results. - // e.g. -P uploaderDestination=gcs://domain-registry-alpha-build-result-test - // - // If not set - the upload will be skipped - destination = uploaderDestination - - // The location of the file containing the OAuth2 Google Cloud credentials. - // - // The file can contain a Service Account key file in JSON format from the - // Google Developers Console or a stored user credential using the format - // supported by the Cloud SDK. - // - // If no file is given - the default credentials are used. - credentialsFile = uploaderCredentialsFile - - // If set to 'yes', each file will be uploaded to GCS in a separate thread. - // This is MUCH faster. - multithreadedUpload = uploaderMultithreadedUpload -} - apply from: 'dependencies.gradle' apply from: 'dependency_lic.gradle' apply from: 'utils.gradle' -// Custom task to run checkLicense in buildSrc, which is not triggered -// by root project tasks. A shell task is used because buildSrc tasks -// cannot be referenced in the same way as tasks from a regular included -// build. -task checkBuildSrcLicense(type:Exec) { - workingDir "${rootDir}/buildSrc" - commandLine '../gradlew', 'checkLicense' -} -tasks.checkLicense.dependsOn(tasks.checkBuildSrcLicense) tasks.build.dependsOn(tasks.checkLicense) // Provide defaults for all of the project properties. @@ -452,9 +419,6 @@ if (verboseTestOutput.toBoolean()) { } task checkDependenciesDotGradle { - def buildSrcDepsFile = File.createTempFile('buildSrc', 'deps') - buildSrcDepsFile.deleteOnExit() - dependsOn createGetBuildSrcDirectDepsTask(buildSrcDepsFile) doLast { Set depsInUse = [] @@ -467,9 +431,7 @@ task checkDependenciesDotGradle { } } } - if (buildSrcDepsFile.exists()) { - depsInUse.addAll(buildSrcDepsFile.readLines()) - } + def unusedDeps = rootProject.dependencyMap.keySet() .findAll { !depsInUse.contains(it) } @@ -486,17 +448,6 @@ task checkDependenciesDotGradle { } tasks.build.dependsOn(tasks.checkDependenciesDotGradle) -def createGetBuildSrcDirectDepsTask(outputFileName) { - return tasks - .create( - "getBuildSrcDeps_${java.util.UUID.randomUUID()}".toString(), - Exec) { - workingDir "${rootDir}/buildSrc" - commandLine '../gradlew', 'exportDependencies', - "-PdependencyExportFile=${outputFileName}" - } -} - rootProject.ext { invokeJavaDiffFormatScript = { action -> def javaHome = project.findProperty('org.gradle.java.home') @@ -507,12 +458,8 @@ rootProject.ext { javaBin = ext.execInBash("which java", rootDir) } println("Running the formatting tool with $javaBin") - def scriptDir = rootDir.path.endsWith('buildSrc') - ? "${rootDir}/../java-format" - : "${rootDir}/java-format" - def workingDir = rootDir.path.endsWith('buildSrc') - ? "${rootDir}/.." - : rootDir + def scriptDir = "${rootDir}/java-format" + def workingDir = rootDir def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh" def pythonExe = getPythonExecutable() diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts deleted file mode 100644 index c04d112cb..000000000 --- a/buildSrc/build.gradle.kts +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import java.io.PrintStream; - -val enableDependencyLocking: String by project -val allowInsecureProtocol: String by project -val allowInsecure = allowInsecureProtocol - -buildscript { - // We need to do this again within "buildscript" because setting it in the - // main script doesn't affect build dependencies. - val enableDependencyLocking: String by project - if (enableDependencyLocking.toBoolean()) { - // Lock application dependencies. - dependencyLocking { - lockAllConfigurations() - } - } -} - -plugins { - // Java static analysis plugins. Keep versions consistent with ../build.gradle - - // We don't anticipate enabling the Gradle lint plugin because they will not support Kotlin - // See https://github.com/nebula-plugins/gradle-lint-plugin/issues/166 - // id 'nebula.lint' version '16.0.2' - id("net.ltgt.errorprone") version "3.1.0" - checkstyle - id("com.diffplug.spotless") version "6.20.0" -} - -checkstyle { - configDirectory.set(file("../config/checkstyle")) -} - -println("enableDependencyLocking is $enableDependencyLocking") -if (enableDependencyLocking.toBoolean()) { - // Lock application dependencies. - dependencyLocking { - lockAllConfigurations() - } -} - -repositories { - val mavenUrl = (project.ext.properties.get("mavenUrl") ?: "") as String - if (mavenUrl.isEmpty()) { - println("Java dependencies: Using Maven central...") - mavenCentral() - google() - } else { - maven { - println("Java dependencies: Using repo ${mavenUrl}...") - url = uri(mavenUrl) - isAllowInsecureProtocol = allowInsecureProtocol == "true" - } - } -} - -apply(from = "../dependencies.gradle") -apply(from = "../dependency_lic.gradle") -apply(from = "../java_common.gradle") - -// 'listenablefuture' is folded into guava since v32. This block is required -// until all transitive dependencies have upgraded past guava v32. -// TODO(periodically): remove this block and see if build succeeds. -configurations.all { - resolutionStrategy - .capabilitiesResolution - .withCapability("com.google.guava:listenablefuture") { - select("com.google.guava:guava:0") - } -} - -project.the()["main"].java { - srcDir("${project.buildDir}/generated/source/apt/main") -} - -dependencies { - val deps = project.ext["dependencyMap"] as Map<*, *> - val implementation by configurations - val testImplementation by configurations - val annotationProcessor by configurations - implementation(deps["com.google.auth:google-auth-library-credentials"]!!) - implementation(deps["com.google.auth:google-auth-library-oauth2-http"]!!) - implementation(deps["com.google.auto.value:auto-value-annotations"]!!) - // implementation(deps["com.google.common.html.types:types"]!!) - implementation(deps["com.google.cloud:google-cloud-core"]!!) - implementation(deps["com.google.cloud:google-cloud-storage"]!!) - implementation(deps["com.google.guava:guava"]!!) - implementation(deps["com.google.protobuf:protobuf-java"]!!) - implementation(deps["com.google.template:soy"]!!) - implementation(deps["org.apache.commons:commons-text"]!!) - annotationProcessor(deps["com.google.auto.value:auto-value"]!!) - testImplementation(deps["com.google.truth:truth"]!!) - testImplementation(deps["org.junit.jupiter:junit-jupiter-api"]!!) - testImplementation(deps["org.junit.jupiter:junit-jupiter-engine"]!!) - testImplementation(deps["org.mockito:mockito-core"]!!) -} - -gradle.projectsEvaluated { - tasks.withType { - options.compilerArgs.add("-Xlint:unchecked") - } -} - -tasks.register("exportDependencies") { - doLast { - project.configurations.forEach { - println("dependency is $it") - // it.dependencies.findAll { - // it.group != null - // }.each { - // output.println("${it.group}:${it.name}") - // } - } - } -} diff --git a/buildSrc/buildscript-gradle.lockfile b/buildSrc/buildscript-gradle.lockfile deleted file mode 100644 index 02c775a54..000000000 --- a/buildSrc/buildscript-gradle.lockfile +++ /dev/null @@ -1,29 +0,0 @@ -# This is a Gradle generated file for dependency locking. -# Manual edits can break the build and are not advised. -# This file is expected to be part of source control. -com.diffplug.durian:durian-collect:1.2.0=classpath -com.diffplug.durian:durian-core:1.2.0=classpath -com.diffplug.durian:durian-io:1.2.0=classpath -com.diffplug.durian:durian-swt.os:4.2.0=classpath -com.diffplug.spotless:com.diffplug.spotless.gradle.plugin:6.20.0=classpath -com.diffplug.spotless:spotless-lib-extra:2.40.0=classpath -com.diffplug.spotless:spotless-lib:2.40.0=classpath -com.diffplug.spotless:spotless-plugin-gradle:6.20.0=classpath -com.googlecode.concurrent-trees:concurrent-trees:2.6.1=classpath -com.googlecode.javaewah:JavaEWAH:1.2.3=classpath -com.squareup.okhttp3:okhttp:4.10.0=classpath -com.squareup.okio:okio-jvm:3.0.0=classpath -com.squareup.okio:okio:3.0.0=classpath -dev.equo.ide:solstice:1.3.1=classpath -net.ltgt.errorprone:net.ltgt.errorprone.gradle.plugin:3.1.0=classpath -net.ltgt.gradle:gradle-errorprone-plugin:3.1.0=classpath -org.eclipse.jgit:org.eclipse.jgit:6.6.0.202305301015-r=classpath -org.eclipse.platform:org.eclipse.osgi:3.18.300=classpath -org.jetbrains.kotlin:kotlin-stdlib-common:1.9.20=classpath -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.8.0=classpath -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.0=classpath -org.jetbrains.kotlin:kotlin-stdlib:1.9.20=classpath -org.jetbrains:annotations:13.0=classpath -org.slf4j:slf4j-api:1.7.36=classpath -org.tukaani:xz:1.9=classpath -empty= diff --git a/buildSrc/gradle.lockfile b/buildSrc/gradle.lockfile deleted file mode 100644 index 98cf9c60f..000000000 --- a/buildSrc/gradle.lockfile +++ /dev/null @@ -1,109 +0,0 @@ -# This is a Gradle generated file for dependency locking. -# Manual edits can break the build and are not advised. -# This file is expected to be part of source control. -aopalliance:aopalliance:1.0=annotationProcessor,buildScriptClasspath,compileClasspath -args4j:args4j:2.0.23=buildScriptClasspath,compileClasspath -com.fasterxml.jackson.core:jackson-core:2.16.1=buildScriptClasspath,compileClasspath -com.fasterxml.jackson:jackson-bom:2.16.1=buildScriptClasspath,compileClasspath -com.github.ben-manes.caffeine:caffeine:3.0.5=annotationProcessor -com.github.kevinstern:software-and-algorithms:1.0=annotationProcessor -com.google.android:annotations:4.1.1.4=buildScriptClasspath -com.google.api-client:google-api-client:2.3.0=buildScriptClasspath,compileClasspath -com.google.api.grpc:gapic-google-cloud-storage-v2:2.35.0-alpha=buildScriptClasspath,compileClasspath -com.google.api.grpc:grpc-google-cloud-storage-v2:2.35.0-alpha=buildScriptClasspath,compileClasspath -com.google.api.grpc:proto-google-cloud-storage-v2:2.35.0-alpha=buildScriptClasspath,compileClasspath -com.google.api.grpc:proto-google-common-protos:2.36.0=buildScriptClasspath,compileClasspath -com.google.api.grpc:proto-google-iam-v1:1.31.0=buildScriptClasspath,compileClasspath -com.google.api:api-common:2.28.0=buildScriptClasspath,compileClasspath -com.google.api:gax-grpc:2.45.0=buildScriptClasspath,compileClasspath -com.google.api:gax-httpjson:2.45.0=buildScriptClasspath,compileClasspath -com.google.api:gax:2.45.0=buildScriptClasspath,compileClasspath -com.google.apis:google-api-services-storage:v1-rev20240209-2.0.0=buildScriptClasspath,compileClasspath -com.google.auth:google-auth-library-credentials:1.23.0=buildScriptClasspath,compileClasspath -com.google.auth:google-auth-library-oauth2-http:1.23.0=buildScriptClasspath,compileClasspath -com.google.auto.service:auto-service-annotations:1.0.1=annotationProcessor -com.google.auto.value:auto-value-annotations:1.10.4=buildScriptClasspath,compileClasspath -com.google.auto.value:auto-value-annotations:1.9=annotationProcessor -com.google.auto.value:auto-value:1.10.4=annotationProcessor -com.google.auto:auto-common:1.2.1=annotationProcessor -com.google.cloud:google-cloud-core-grpc:2.35.0=buildScriptClasspath,compileClasspath -com.google.cloud:google-cloud-core-http:2.35.0=buildScriptClasspath,compileClasspath -com.google.cloud:google-cloud-core:2.35.0=buildScriptClasspath,compileClasspath -com.google.cloud:google-cloud-storage:2.35.0=buildScriptClasspath,compileClasspath -com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,buildScriptClasspath,compileClasspath -com.google.code.gson:gson:2.10.1=buildScriptClasspath,compileClasspath -com.google.common.html.types:types:1.0.6=buildScriptClasspath,compileClasspath -com.google.errorprone:error_prone_annotation:2.23.0=annotationProcessor -com.google.errorprone:error_prone_annotations:2.23.0=annotationProcessor -com.google.errorprone:error_prone_annotations:2.26.1=buildScriptClasspath,compileClasspath -com.google.errorprone:error_prone_check_api:2.23.0=annotationProcessor -com.google.errorprone:error_prone_core:2.23.0=annotationProcessor -com.google.errorprone:error_prone_type_annotations:2.23.0=annotationProcessor -com.google.escapevelocity:escapevelocity:0.9.1=buildScriptClasspath,compileClasspath -com.google.guava:failureaccess:1.0.1=annotationProcessor -com.google.guava:failureaccess:1.0.2=buildScriptClasspath,compileClasspath -com.google.guava:guava-parent:32.1.1-jre=annotationProcessor -com.google.guava:guava:32.1.1-jre=annotationProcessor -com.google.guava:guava:33.1.0-jre=buildScriptClasspath,compileClasspath -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=buildScriptClasspath,compileClasspath -com.google.http-client:google-http-client-apache-v2:1.44.1=buildScriptClasspath,compileClasspath -com.google.http-client:google-http-client-appengine:1.44.1=buildScriptClasspath,compileClasspath -com.google.http-client:google-http-client-gson:1.44.1=buildScriptClasspath,compileClasspath -com.google.http-client:google-http-client-jackson2:1.44.1=buildScriptClasspath,compileClasspath -com.google.http-client:google-http-client:1.44.1=buildScriptClasspath,compileClasspath -com.google.inject.extensions:guice-multibindings:4.1.0=buildScriptClasspath,compileClasspath -com.google.inject:guice:4.1.0=buildScriptClasspath,compileClasspath -com.google.inject:guice:5.1.0=annotationProcessor -com.google.j2objc:j2objc-annotations:2.8=buildScriptClasspath -com.google.j2objc:j2objc-annotations:3.0.0=compileClasspath -com.google.jsinterop:jsinterop-annotations:1.0.1=buildScriptClasspath,compileClasspath -com.google.oauth-client:google-oauth-client:1.35.0=buildScriptClasspath,compileClasspath -com.google.protobuf:protobuf-java-util:3.25.2=buildScriptClasspath,compileClasspath -com.google.protobuf:protobuf-java:3.19.6=annotationProcessor -com.google.protobuf:protobuf-java:3.25.2=buildScriptClasspath,compileClasspath -com.google.re2j:re2j:1.7=buildScriptClasspath -com.google.template:soy:2021-02-01=buildScriptClasspath,compileClasspath -com.ibm.icu:icu4j:57.1=buildScriptClasspath,compileClasspath -commons-codec:commons-codec:1.16.1=buildScriptClasspath,compileClasspath -io.github.eisop:dataflow-errorprone:3.34.0-eisop1=annotationProcessor -io.github.java-diff-utils:java-diff-utils:4.12=annotationProcessor -io.grpc:grpc-alts:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-api:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-auth:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-context:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-core:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-googleapis:1.62.2=buildScriptClasspath -io.grpc:grpc-grpclb:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-inprocess:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-netty-shaded:1.62.2=buildScriptClasspath -io.grpc:grpc-protobuf-lite:1.62.2=buildScriptClasspath -io.grpc:grpc-protobuf:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-rls:1.62.2=buildScriptClasspath -io.grpc:grpc-services:1.62.2=buildScriptClasspath -io.grpc:grpc-stub:1.62.2=buildScriptClasspath,compileClasspath -io.grpc:grpc-util:1.62.2=buildScriptClasspath -io.grpc:grpc-xds:1.62.2=buildScriptClasspath -io.opencensus:opencensus-api:0.31.1=buildScriptClasspath,compileClasspath -io.opencensus:opencensus-contrib-http-util:0.31.1=buildScriptClasspath,compileClasspath -io.opencensus:opencensus-proto:0.2.0=buildScriptClasspath -io.perfmark:perfmark-api:0.27.0=buildScriptClasspath -javax.annotation:javax.annotation-api:1.3.2=buildScriptClasspath,compileClasspath -javax.annotation:jsr250-api:1.0=buildScriptClasspath,compileClasspath -javax.inject:javax.inject:1=annotationProcessor,buildScriptClasspath,compileClasspath -org.apache.commons:commons-lang3:3.13.0=buildScriptClasspath,compileClasspath -org.apache.commons:commons-text:1.11.0=buildScriptClasspath,compileClasspath -org.apache.httpcomponents:httpclient:4.5.14=buildScriptClasspath,compileClasspath -org.apache.httpcomponents:httpcore:4.4.16=buildScriptClasspath,compileClasspath -org.checkerframework:checker-qual:3.33.0=annotationProcessor -org.checkerframework:checker-qual:3.42.0=buildScriptClasspath,compileClasspath -org.codehaus.mojo:animal-sniffer-annotations:1.23=buildScriptClasspath -org.conscrypt:conscrypt-openjdk-uber:2.5.2=buildScriptClasspath,compileClasspath -org.json:json:20160212=buildScriptClasspath,compileClasspath -org.ow2.asm:asm-analysis:7.0=buildScriptClasspath,compileClasspath -org.ow2.asm:asm-commons:7.0=buildScriptClasspath,compileClasspath -org.ow2.asm:asm-tree:7.0=buildScriptClasspath,compileClasspath -org.ow2.asm:asm-util:7.0=buildScriptClasspath,compileClasspath -org.ow2.asm:asm:7.0=buildScriptClasspath,compileClasspath -org.pcollections:pcollections:3.1.4=annotationProcessor -org.threeten:threetenbp:1.6.8=buildScriptClasspath,compileClasspath -empty= diff --git a/buildSrc/gradle.properties b/buildSrc/gradle.properties deleted file mode 100644 index c7ef7adbd..000000000 --- a/buildSrc/gradle.properties +++ /dev/null @@ -1,3 +0,0 @@ -pluginsUrl=https://plugins.gradle.org/m2/ -allowInsecureProtocol= -enableDependencyLocking=true diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle deleted file mode 100644 index a85c9dbc9..000000000 --- a/buildSrc/settings.gradle +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Alias this since it collides with the closure variable name -def allowInsecure = allowInsecureProtocol - -if (!pluginsUrl.isEmpty()) { - pluginManagement { - repositories { - maven { - url pluginsUrl - allowInsecureProtocol = allowInsecure == "true" - } - } - } -} else { - println "Plugins: Using default repo..." -} diff --git a/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java b/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java deleted file mode 100644 index 629d6003a..000000000 --- a/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gradle.plugin; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap; -import static com.google.common.io.Resources.getResource; -import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSetMultimap; -import com.google.common.html.types.TrustedResourceUrls; -import com.google.template.soy.SoyFileSet; -import com.google.template.soy.tofu.SoyTofu; -import google.registry.gradle.plugin.ProjectData.TaskData; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -/** - * Creates the files for a web-page summary of a given {@link ProjectData}. - * - *

The main job of this class is rendering a tailored cover page that includes information about - * the project and any task that ran. - * - *

It returns all the files that need uploading for the cover page to work. This includes any - * report and log files linked to in the ProjectData, as well as a cover page (and associated - * resources such as CSS files). - */ -final class CoverPageGenerator { - - /** List of all resource files that will be uploaded as-is. */ - private static final ImmutableSet STATIC_RESOURCE_FILES = - ImmutableSet.of(Paths.get("css", "style.css")); - /** Name of the entry-point file that will be created. */ - private static final Path ENTRY_POINT = Paths.get("index.html"); - - private final ProjectData projectData; - private final ImmutableSetMultimap tasksByState; - /** - * The compiled SOY files. - * - *

Will be generated only when actually needed, because it takes a while to compile and we - * don't want that to happen unless we actually use it. - */ - private SoyTofu tofu = null; - - CoverPageGenerator(ProjectData projectData) { - this.projectData = projectData; - this.tasksByState = - projectData.tasks().stream().collect(toImmutableSetMultimap(TaskData::state, task -> task)); - } - - /** - * Returns all the files that need uploading for the cover page to work. - * - *

This includes all the report files as well, to make sure that the link works. - */ - FilesWithEntryPoint getFilesToUpload() { - ImmutableMap.Builder> builder = new ImmutableMap.Builder<>(); - // Add all the static resource pages - STATIC_RESOURCE_FILES.stream().forEach(file -> builder.put(file, resourceLoader(file))); - // Create the cover page - // Note that the ByteArraySupplier here is lazy - the createCoverPage function is only called - // when the resulting Supplier's get function is called. - builder.put(ENTRY_POINT, toByteArraySupplier(this::createCoverPage)); - // Add all the files from the tasks - tasksByState.values().stream() - .flatMap(task -> task.reports().values().stream()) - .forEach(reportFiles -> builder.putAll(reportFiles.files())); - // Add the logs of every test - tasksByState.values().stream() - .filter(task -> task.log().isPresent()) - .forEach(task -> builder.put(getLogPath(task), task.log().get())); - - return FilesWithEntryPoint.create(builder.build(), ENTRY_POINT); - } - - /** Renders the cover page. */ - private String createCoverPage() { - return getTofu() - .newRenderer("google.registry.gradle.plugin.coverPage") - .setData(getSoyData()) - .render(); - } - - /** Converts the projectData and all taskData into all the data the soy template needs. */ - private ImmutableMap getSoyData() { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - - TaskData.State state = - tasksByState.containsKey(TaskData.State.FAILURE) - ? TaskData.State.FAILURE - : TaskData.State.SUCCESS; - String title = - state != TaskData.State.FAILURE - ? "Success!" - : "Failed: " - + tasksByState.get(state).stream() - .map(TaskData::uniqueName) - .collect(Collectors.joining(", ")); - - builder.put("projectState", state.toString()); - builder.put("title", title); - builder.put("cssFiles", ImmutableSet.of(TrustedResourceUrls.fromConstant("css/style.css"))); - builder.put("invocation", getInvocation()); - builder.put("tasksByState", getTasksByStateSoyData()); - return builder.build(); - } - - /** - * Returns a soy-friendly map from the TaskData.State to the task itslef. - * - *

The key order in the resulting map is always the same (the order from the enum definition) - * no matter the key order in the original tasksByState map. - */ - private ImmutableMap getTasksByStateSoyData() { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - - // We go over the States in the order they are defined rather than the order in which they - // happen to be in the tasksByState Map. - // - // That way we guarantee a consistent order. - for (TaskData.State state : TaskData.State.values()) { - builder.put( - state.toString(), - tasksByState.get(state).stream() - .map(task -> taskDataToSoy(task)) - .collect(toImmutableList())); - } - - return builder.build(); - } - - /** returns a soy-friendly version of the given task data. */ - static ImmutableMap taskDataToSoy(TaskData task) { - // Note that all instances of File.separator are replaced with forward slashes so that we can - // generate a valid href on Windows. - return new ImmutableMap.Builder() - .put("uniqueName", task.uniqueName()) - .put("description", task.description()) - .put( - "log", - task.log().isPresent() ? getLogPath(task).toString().replace(File.separator, "/") : "") - .put( - "reports", - task.reports().entrySet().stream() - .collect( - toImmutableMap( - Map.Entry::getKey, - entry -> - entry.getValue().files().isEmpty() - ? "" - : entry - .getValue() - .entryPoint() - .toString() - .replace(File.separator, "/")))) - .build(); - } - - private String getInvocation() { - StringBuilder builder = new StringBuilder(); - builder.append("./gradlew"); - projectData.tasksRequested().forEach(task -> builder.append(" ").append(task)); - projectData - .projectProperties() - .forEach((key, value) -> builder.append(String.format(" -P %s=%s", key, value))); - return builder.toString(); - } - - /** Returns a lazily created soy renderer */ - private SoyTofu getTofu() { - if (tofu == null) { - tofu = - SoyFileSet.builder() - .add(getResource(CoverPageGenerator.class, "soy/coverpage.soy")) - .build() - .compileToTofu(); - } - return tofu; - } - - private static Path getLogPath(TaskData task) { - // We replace colons with dashes so that the resulting filename is always valid, even in - // Windows. As a dash is not a valid character in Java identifies, a task name cannot include - // it, so the uniqueness of the name is perserved. - return Paths.get("logs", task.uniqueName().replace(":", "-") + ".log"); - } - - private static Supplier resourceLoader(Path path) { - return toByteArraySupplier(getResource(CoverPageGenerator.class, path.toString())); - } -} diff --git a/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java b/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java deleted file mode 100644 index f8f098b38..000000000 --- a/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gradle.plugin; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableMap; -import java.nio.file.Path; -import java.util.function.Supplier; - -/** - * Holds a set of files with a browser-friendly entry point to those files. - * - *

The file data is lazily generated. - * - *

If there is at least one file, it's guaranteed that the entry point is one of these files. - */ -@AutoValue -abstract class FilesWithEntryPoint { - - /** - * All files that are part of this report, keyed from their path to a supplier of their content. - * - *

The reason we use a supplier instead of loading the content is in case the content is very - * large... - * - *

Also, no point in doing IO before we need it! - */ - abstract ImmutableMap> files(); - - /** - * The file that gives access (links...) to all the data in the report. - * - *

Guaranteed to be a key in {@link #files} if and only if files isn't empty. - */ - abstract Path entryPoint(); - - static FilesWithEntryPoint create(ImmutableMap> files, Path entryPoint) { - checkArgument(files.isEmpty() || files.containsKey(entryPoint)); - return new AutoValue_FilesWithEntryPoint(files, entryPoint); - } - - static FilesWithEntryPoint createSingleFile(Path path, Supplier data) { - return create(ImmutableMap.of(path, data), path); - } -} diff --git a/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java b/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java deleted file mode 100644 index 48adbb68c..000000000 --- a/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gradle.plugin; - -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.collect.Iterables.getOnlyElement; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Streams; -import com.google.common.io.Files; -import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.URL; -import java.nio.file.Path; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -/** Utility functions used in the GCS plugin. */ -final class GcsPluginUtils { - - private static final ImmutableMap EXTENSION_TO_CONTENT_TYPE = - new ImmutableMap.Builder() - .put("html", "text/html") - .put("htm", "text/html") - .put("log", "text/plain") - .put("txt", "text/plain") - .put("css", "text/css") - .put("xml", "text/xml") - .put("zip", "application/zip") - .put("js", "text/javascript") - .build(); - - private static final String DEFAULT_CONTENT_TYPE = "application/octet-stream"; - - static Path toNormalizedPath(File file) { - return file.toPath().toAbsolutePath().normalize(); - } - - static Path toNormalizedPath(Path file) { - return file.toAbsolutePath().normalize(); - } - - static String getContentType(String fileName) { - return EXTENSION_TO_CONTENT_TYPE.getOrDefault( - Files.getFileExtension(fileName), DEFAULT_CONTENT_TYPE); - } - - static void uploadFileToGcs( - Storage storage, String bucket, Path path, Supplier dataSupplier) { - // Replace Windows file separators with forward slashes. - String filename = path.toString().replace(File.separator, "/"); - storage.create( - BlobInfo.newBuilder(bucket, filename).setContentType(getContentType(filename)).build(), - dataSupplier.get()); - } - - static void uploadFilesToGcsMultithread( - Storage storage, String bucket, Path folder, Map> files) { - ImmutableMap.Builder threads = new ImmutableMap.Builder<>(); - files.forEach( - (path, dataSupplier) -> { - Thread thread = - new Thread( - () -> uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier)); - thread.start(); - threads.put(path, thread); - }); - threads - .build() - .forEach( - (path, thread) -> { - try { - thread.join(); - } catch (InterruptedException e) { - System.out.format("Upload of %s interrupted", path); - } - }); - } - - static Supplier toByteArraySupplier(String data) { - return () -> data.getBytes(UTF_8); - } - - static Supplier toByteArraySupplier(Supplier dataSupplier) { - return () -> dataSupplier.get().getBytes(UTF_8); - } - - static Supplier toByteArraySupplier(File file) { - return () -> { - try { - return Files.toByteArray(file); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - } - - static Supplier toByteArraySupplier(URL url) { - return () -> { - try { - return Resources.toByteArray(url); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - } - - /** - * Reads all the files generated by a Report into a FilesWithEntryPoint object. - * - *

Every FilesWithEntryPoint must have a single link "entry point" that gives users access to - * all the files. If the report generated just one file - we will just link to that file. - * - *

However, if the report generated more than one file - the only thing we can safely do is to - * zip all the files and link to the zip file. - * - *

As an alternative to using a zip file, we allow the caller to supply an optional "entry - * point" file that will link to all the other files. If that file is given and is "appropriate" - * (exists and is in the correct location) - we will upload all the report files "as is" and link - * to the entry file. - * - * @param destination the location of the output. Either a file or a directory. If a directory - - * then all the files inside that directory are the outputs we're looking for. - * @param entryPointHint If present - a hint to what the entry point to this directory tree is. - * Will only be used if all of the following apply: (a) {@code - * destination.isDirectory()==true}, (b) there are 2 or more files in the {@code destination} - * directory, and (c) {@code entryPointHint.get()} is one of the files nested inside of the - * {@code destination} directory. - */ - static FilesWithEntryPoint readFilesWithEntryPoint( - File destination, Optional entryPointHint, Path rootDir) { - - Path destinationPath = rootDir.relativize(toNormalizedPath(destination)); - - if (destination.isFile()) { - // The destination is a single file - find its root, and add this single file to the - // FilesWithEntryPoint. - return FilesWithEntryPoint.createSingleFile( - destinationPath, toByteArraySupplier(destination)); - } - - if (!destination.isDirectory()) { - // This isn't a file nor a directory - so it doesn't exist! Return empty FilesWithEntryPoint - return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath); - } - - // The destination is a directory - find all the actual files first - ImmutableMap> files = - Streams.stream(Files.fileTraverser().depthFirstPreOrder(destination)) - .filter(File::isFile) - .collect( - toImmutableMap( - file -> rootDir.relativize(toNormalizedPath(file)), - GcsPluginUtils::toByteArraySupplier)); - - if (files.isEmpty()) { - // The directory exists, but is empty. Return empty FilesWithEntryPoint - return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath); - } - - if (files.size() == 1) { - // We got a directory, but it only has a single file. We can link to that. - return FilesWithEntryPoint.create(files, getOnlyElement(files.keySet())); - } - - // There are multiple files in the report! We need to check the entryPointHint - Optional entryPointPath = - entryPointHint.map(file -> rootDir.relativize(toNormalizedPath(file))); - - if (entryPointPath.isPresent() && files.containsKey(entryPointPath.get())) { - // We were given the entry point! Use it! - return FilesWithEntryPoint.create(files, entryPointPath.get()); - } - - // We weren't given an appropriate entry point. But we still need a single link to all this data - // - so we'll zip it and just host a single file. - Path zipFilePath = destinationPath.resolve(destinationPath.getFileName().toString() + ".zip"); - return FilesWithEntryPoint.createSingleFile(zipFilePath, createZippedByteArraySupplier(files)); - } - - static Supplier createZippedByteArraySupplier(Map> files) { - return () -> zipFiles(files); - } - - private static byte[] zipFiles(Map> files) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - try (ZipOutputStream zip = new ZipOutputStream(output)) { - for (Path path : files.keySet()) { - zip.putNextEntry(new ZipEntry(path.toString())); - zip.write(files.get(path).get()); - zip.closeEntry(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return output.toByteArray(); - } - - private GcsPluginUtils() {} -} diff --git a/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java b/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java deleted file mode 100644 index 7a8096c70..000000000 --- a/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gradle.plugin; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; - -/** - * All the data of a root Gradle project. - * - *

This is basically all the "relevant" data from a Gradle Project, arranged in an immutable and - * more convenient way. - */ -@AutoValue -abstract class ProjectData { - - abstract String name(); - - abstract String description(); - - abstract String gradleVersion(); - - abstract ImmutableMap projectProperties(); - - abstract ImmutableMap systemProperties(); - - abstract ImmutableSet tasksRequested(); - - abstract ImmutableSet tasks(); - - abstract Builder toBuilder(); - - static Builder builder() { - return new AutoValue_ProjectData.Builder(); - } - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setName(String name); - - abstract Builder setDescription(String description); - - abstract Builder setGradleVersion(String gradleVersion); - - abstract Builder setProjectProperties(Map projectProperties); - - abstract Builder setSystemProperties(Map systemProperties); - - abstract Builder setTasksRequested(Iterable tasksRequested); - - abstract ImmutableSet.Builder tasksBuilder(); - - Builder addTask(TaskData task) { - tasksBuilder().add(task); - return this; - } - - abstract ProjectData build(); - } - - /** - * Relevant data to a single Task's. - * - *

Some Tasks are also "Reporting", meaning they create file outputs we want to upload in - * various formats. The format that interests us the most is "html", as that's nicely browsable, - * but they might also have other formats. - */ - @AutoValue - abstract static class TaskData { - - enum State { - /** The task has failed for some reason. */ - FAILURE, - /** The task was actually run and has finished successfully. */ - SUCCESS, - /** The task was up-to-date and successful, and hence didn't need to run again. */ - UP_TO_DATE - } - - abstract String uniqueName(); - - abstract String description(); - - abstract State state(); - - abstract Optional> log(); - - /** - * Returns the FilesWithEntryPoint for every report, keyed on the report type. - * - *

The "html" report type is the most interesting, but there are other report formats. - */ - abstract ImmutableMap reports(); - - abstract Builder toBuilder(); - - static Builder builder() { - return new AutoValue_ProjectData_TaskData.Builder(); - } - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setUniqueName(String name); - - abstract Builder setDescription(String description); - - abstract Builder setState(State state); - - abstract Builder setLog(Supplier log); - - abstract ImmutableMap.Builder reportsBuilder(); - - Builder putReport(String type, FilesWithEntryPoint reportFiles) { - reportsBuilder().put(type, reportFiles); - return this; - } - - abstract TaskData build(); - } - } -} diff --git a/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java b/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java deleted file mode 100644 index 56ff1445d..000000000 --- a/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java +++ /dev/null @@ -1,311 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gradle.plugin; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Strings.isNullOrEmpty; -import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint; -import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; -import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath; -import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs; -import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread; - -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import com.google.common.collect.ImmutableMap; -import com.google.common.io.Files; -import google.registry.gradle.plugin.ProjectData.TaskData; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Optional; -import java.util.function.BiConsumer; -import java.util.function.Supplier; -import org.gradle.api.DefaultTask; -import org.gradle.api.Project; -import org.gradle.api.Task; -import org.gradle.api.reporting.DirectoryReport; -import org.gradle.api.reporting.Report; -import org.gradle.api.reporting.ReportContainer; -import org.gradle.api.reporting.Reporting; -import org.gradle.api.tasks.TaskAction; - -/** A task that uploads the Reports generated by other tasks to GCS. */ -public class ReportUploader extends DefaultTask { - - private static final SecureRandom SECURE_RANDOM = new SecureRandom(); - private static final ImmutableMap> UPLOAD_FUNCTIONS = - ImmutableMap.of( - "file://", ReportUploader::saveResultsToLocalFolder, - "gcs://", ReportUploader::uploadResultsToGcs); - - private final ArrayList tasks = new ArrayList<>(); - private final HashMap logs = new HashMap<>(); - private Project project; - - private String destination = null; - private String credentialsFile = null; - private String multithreadedUpload = null; - - /** - * Sets the destination of the reports. - * - *

Currently supports two types of destinations: - * - *

    - *
  • file://[absulute local path], e.g. file:///tmp/buildOutputs/ - *
  • gcs://[bucket name]/[optional path], e.g. gcs://my-bucket/buildOutputs/ - *
- */ - public void setDestination(String destination) { - this.destination = destination; - } - - public void setCredentialsFile(String credentialsFile) { - this.credentialsFile = credentialsFile; - } - - public void setMultithreadedUpload(String multithreadedUpload) { - this.multithreadedUpload = multithreadedUpload; - } - - /** Converts the given Gradle Project into a ProjectData. */ - private ProjectData createProjectData() { - ProjectData.Builder builder = - ProjectData.builder() - .setName(project.getPath() + project.getName()) - .setDescription( - Optional.ofNullable(project.getDescription()).orElse("[No description available]")) - .setGradleVersion(project.getGradle().getGradleVersion()) - .setProjectProperties(project.getGradle().getStartParameter().getProjectProperties()) - .setSystemProperties(project.getGradle().getStartParameter().getSystemPropertiesArgs()) - .setTasksRequested(project.getGradle().getStartParameter().getTaskNames()); - - Path rootDir = toNormalizedPath(project.getRootDir()); - tasks.stream() - .filter(task -> task.getState().getExecuted() || task.getState().getUpToDate()) - .map(task -> createTaskData(task, rootDir)) - .forEach(builder.tasksBuilder()::add); - return builder.build(); - } - - /** - * Converts a Gradle Task into a TaskData. - * - * @param rootDir the root directory of the main Project - used to get the relative path of any - * Task files. - */ - private TaskData createTaskData(Task task, Path rootDir) { - TaskData.State state = - task.getState().getFailure() != null - ? TaskData.State.FAILURE - : task.getState().getUpToDate() ? TaskData.State.UP_TO_DATE : TaskData.State.SUCCESS; - String log = logs.get(task.getPath()).toString(); - - TaskData.Builder builder = - TaskData.builder() - .setState(state) - .setUniqueName(task.getPath()) - .setDescription( - Optional.ofNullable(task.getDescription()).orElse("[No description available]")); - if (!log.isEmpty()) { - builder.setLog(toByteArraySupplier(log)); - } - - Reporting> reporting = asReporting(task); - - if (reporting != null) { - // This Task is also a Reporting task! It has a destination file/directory for every supported - // format. - // Add the files for each of the formats into the ReportData. - reporting - .getReports() - .getAsMap() - .forEach( - (type, report) -> { - File destination = report.getOutputLocation().get().getAsFile(); - // The destination could be a file, or a directory. If it's a directory - the Report - // could have created multiple files - and we need to know to which one of those to - // link. - // - // If we're lucky, whoever implemented the Report made sure to extend - // DirectoryReport, which gives us the entry point to all the files. - // - // This isn't guaranteed though, as it depends on the implementer. - Optional entryPointHint = - destination.isDirectory() && (report instanceof DirectoryReport) - ? Optional.ofNullable(((DirectoryReport) report).getEntryPoint()) - : Optional.empty(); - builder - .reportsBuilder() - .put(type, readFilesWithEntryPoint(destination, entryPointHint, rootDir)); - }); - } - return builder.build(); - } - - private FilesWithEntryPoint generateFilesToUpload() { - ProjectData projectData = createProjectData(); - CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData); - return coverPageGenerator.getFilesToUpload(); - } - - @TaskAction - void uploadResults() { - System.out.format("ReportUploader: destination= '%s'\n", destination); - - try { - - if (isNullOrEmpty(destination)) { - System.out.format("ReportUploader: no destination given, skipping...\n"); - return; - } - - for (String key : UPLOAD_FUNCTIONS.keySet()) { - if (destination.startsWith(key)) { - UPLOAD_FUNCTIONS.get(key).accept(this, destination.substring(key.length())); - return; - } - } - System.out.format( - "ReportUploader: given destination '%s' doesn't start with one of %s." - + " Defaulting to saving in /tmp\n", - destination, UPLOAD_FUNCTIONS.keySet()); - saveResultsToLocalFolder("/tmp/"); - } catch (Throwable e) { - System.out.format("ReportUploader: Encountered error %s\n", e); - e.printStackTrace(System.out); - System.out.format("ReportUploader: skipping upload\n"); - } - } - - private void saveResultsToLocalFolder(String absoluteFolderName) { - Path folder = Paths.get(absoluteFolderName, createUniqueFolderName()); - checkArgument( - folder.isAbsolute(), - "Local files destination must be an absolute path, but is %s", - absoluteFolderName); - FilesWithEntryPoint filesToUpload = generateFilesToUpload(); - System.out.format( - "ReportUploader: going to save %s files to %s\n", filesToUpload.files().size(), folder); - filesToUpload - .files() - .forEach((path, dataSupplier) -> saveFile(folder.resolve(path), dataSupplier)); - System.out.format( - "ReportUploader: report saved to file://%s\n", folder.resolve(filesToUpload.entryPoint())); - } - - private void saveFile(Path path, Supplier dataSupplier) { - File dir = path.getParent().toFile(); - if (!dir.isDirectory()) { - checkState(dir.mkdirs(), "Couldn't create directory %s", dir); - } - try { - Files.write(dataSupplier.get(), path.toFile()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private void uploadResultsToGcs(String destination) { - checkArgument( - !destination.isEmpty(), "destination must include at least the bucket name, but is empty"); - Path bucketWithFolder = Paths.get(destination, createUniqueFolderName()); - String bucket = bucketWithFolder.getName(0).toString(); - Path folder = bucketWithFolder.subpath(1, bucketWithFolder.getNameCount()); - - StorageOptions.Builder storageOptions = StorageOptions.newBuilder(); - if (!isNullOrEmpty(credentialsFile)) { - try { - storageOptions.setCredentials( - GoogleCredentials.fromStream(new FileInputStream(credentialsFile))); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - Storage storage = storageOptions.build().getService(); - - FilesWithEntryPoint filesToUpload = generateFilesToUpload(); - - System.out.format( - "ReportUploader: going to upload %s files to %s/%s\n", - filesToUpload.files().size(), bucket, folder); - if ("yes".equals(multithreadedUpload)) { - System.out.format("ReportUploader: multi-threaded upload\n"); - uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload.files()); - } else { - System.out.format("ReportUploader: single threaded upload\n"); - filesToUpload - .files() - .forEach( - (path, dataSupplier) -> { - System.out.format("ReportUploader: Uploading %s\n", path); - uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier); - }); - } - System.out.format( - "ReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n", - bucket, folder.resolve(filesToUpload.entryPoint())); - } - - void setProject(Project project) { - this.project = project; - - for (Project subProject : project.getAllprojects()) { - subProject.getTasks().all(this::addTask); - } - } - - private void addTask(Task task) { - if (task instanceof ReportUploader) { - return; - } - tasks.add(task); - StringBuilder log = new StringBuilder(); - checkArgument( - !logs.containsKey(task.getPath()), - "Multiple tasks with the same .getPath()=%s", - task.getPath()); - logs.put(task.getPath(), log); - task.getLogging().addStandardOutputListener(output -> log.append(output)); - task.getLogging().addStandardErrorListener(output -> log.append(output)); - task.finalizedBy(this); - } - - @SuppressWarnings("unchecked") - private static Reporting> asReporting(Task task) { - if (task instanceof Reporting) { - return (Reporting>) task; - } - return null; - } - - private String createUniqueFolderName() { - return String.format( - "%h-%h-%h-%h", - SECURE_RANDOM.nextInt(), - SECURE_RANDOM.nextInt(), - SECURE_RANDOM.nextInt(), - SECURE_RANDOM.nextInt()); - } -} diff --git a/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java b/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java deleted file mode 100644 index aad24b024..000000000 --- a/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gradle.plugin; - -import org.gradle.api.Plugin; -import org.gradle.api.Project; - -/** - * Plugin setting up the ReportUploader task. - * - *

It goes over all the tasks in a project and pass them on to the ReportUploader task for set - * up. - * - *

Note that since we're passing in all the projects' tasks - this includes the ReportUploader - * itself! It's up to the ReportUploader to take care of not having "infinite loops" caused by - * waiting for itself to end before finishing. - */ -public class ReportUploaderPlugin implements Plugin { - - @Override - public void apply(Project project) { - ReportUploader reportUploader = - project - .getTasks() - .create( - "reportUploader", - ReportUploader.class, - task -> { - task.setDescription("Uploads the reports to GCS bucket"); - task.setGroup("uploads"); - }); - - reportUploader.setProject(project); - } -} diff --git a/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css b/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css deleted file mode 100644 index 384948449..000000000 --- a/buildSrc/src/main/resources/google/registry/gradle/plugin/css/style.css +++ /dev/null @@ -1,27 +0,0 @@ -body { - font-family: sans-serif; -} -.task_state_SUCCESS { - color: green; -} -.task_state_FAILURE { - color: red; -} -.task_name { - display: block; - font-size: larger; - font-weight: bold; -} -.task_description { - display: block; - margin-left: 1em; - color: gray; -} -.report_links { - margin-left: 1em; -} -.report_link_broken { - text-decoration: line-through; - color: gray; -} - diff --git a/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy b/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy deleted file mode 100644 index 30eb5083b..000000000 --- a/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -{namespace google.registry.gradle.plugin} - -{template .coverPage} - {@param title: string} - {@param cssFiles: list} - {@param projectState: string} - {@param invocation: string} - {@param tasksByState: map]>>} - - {$title} - {for $cssFile in $cssFiles} - - {/for} - -

-

{$title}

- - Build results for {$invocation} - - - {for $taskState in mapKeys($tasksByState)} - {if length($tasksByState[$taskState]) > 0} - {call .tasksOfState} - {param state: $taskState /} - {param tasks: $tasksByState[$taskState] /} - {/call} - {/if} - {/for} - -
- -{/template} - -{template .tasksOfState} - {@param state: string} - {@param tasks: list<[uniqueName: string, description: string, log: string, reports: map]>} - -
-

{$state}

- // Place the tasks with actual reports first, since those are more likely to be useful - {for $task in $tasks} - {if length(mapKeys($task.reports)) > 0} - {call .task} - {param task: $task /} - {/call} - {/if} - {/for} - // Followup with reports without links - {for $task in $tasks} - {if length(mapKeys($task.reports)) == 0} - {call .task} - {param task: $task /} - {/call} - {/if} - {/for} -
-{/template} - -{template .task} - {@param task: [uniqueName: string, description: string, log: string, reports: map]} - {call .taskInternal} - {param uniqueName: $task.uniqueName /} - {param description: $task.description /} - {param log: $task.log /} - {param reports: $task.reports /} - {/call} -{/template} - -{template .taskInternal} - {@param uniqueName: string} - {@param description: string} - {@param log: string} - {@param reports: map} - -
- {$uniqueName} - {$description} - - {if $log} - [log] - {else} - [log] - {/if} - {for $type in mapKeys($reports)} - {if $reports[$type]} - [{$type}] - {else} - [{$type}] - {/if} - {/for} - -
-{/template} diff --git a/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java b/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java deleted file mode 100644 index 01d1ec164..000000000 --- a/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gradle.plugin; - -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.truth.Truth.assertThat; -import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import google.registry.gradle.plugin.ProjectData.TaskData; -import java.io.File; -import java.nio.file.Paths; -import org.junit.jupiter.api.Test; - -/** Tests for {@link CoverPageGenerator} */ -final class CoverPageGeneratorTest { - - private static final ProjectData EMPTY_PROJECT = - ProjectData.builder() - .setName("project-name") - .setDescription("project-description") - .setGradleVersion("gradle-version") - .setProjectProperties(ImmutableMap.of("key", "value")) - .setSystemProperties(ImmutableMap.of()) - .setTasksRequested(ImmutableSet.of(":a:task1", ":a:task2")) - .build(); - - private static final TaskData EMPTY_TASK_SUCCESS = - TaskData.builder() - .setUniqueName("task-success") - .setDescription("a successful task") - .setState(TaskData.State.SUCCESS) - .build(); - - private static final TaskData EMPTY_TASK_FAILURE = - TaskData.builder() - .setUniqueName("task-failure") - .setDescription("a failed task") - .setState(TaskData.State.FAILURE) - .build(); - - private static final TaskData EMPTY_TASK_UP_TO_DATE = - TaskData.builder() - .setUniqueName("task-up-to-date") - .setDescription("an up-to-date task") - .setState(TaskData.State.UP_TO_DATE) - .build(); - - private static final Joiner filenameJoiner = Joiner.on(File.separator); - - private ImmutableMap getGeneratedFiles(ProjectData project) { - CoverPageGenerator coverPageGenerator = new CoverPageGenerator(project); - FilesWithEntryPoint files = coverPageGenerator.getFilesToUpload(); - return files.files().entrySet().stream() - .collect( - toImmutableMap( - entry -> entry.getKey().toString(), - entry -> new String(entry.getValue().get(), UTF_8))); - } - - private String getContentOfGeneratedFile(ProjectData project, String expectedPath) { - ImmutableMap files = getGeneratedFiles(project); - assertThat(files).containsKey(expectedPath); - return files.get(expectedPath); - } - - private String getCoverPage(ProjectData project) { - return getContentOfGeneratedFile(project, "index.html"); - } - - @Test - void testGetFilesToUpload_entryPoint_isIndexHtml() { - CoverPageGenerator coverPageGenerator = new CoverPageGenerator(EMPTY_PROJECT); - assertThat(coverPageGenerator.getFilesToUpload().entryPoint()) - .isEqualTo(Paths.get("index.html")); - } - - @Test - void testGetFilesToUpload_containsEntryFile() { - String content = getContentOfGeneratedFile(EMPTY_PROJECT, "index.html"); - assertThat(content) - .contains( - "./gradlew :a:task1 :a:task2 -P key=value"); - } - - @Test - void testCoverPage_showsFailedTask() { - String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_FAILURE).build()); - assertThat(content).contains("task-failure"); - assertThat(content).contains("

FAILURE

"); - assertThat(content).doesNotContain("

SUCCESS

"); - assertThat(content).doesNotContain("

UP_TO_DATE

"); - } - - @Test - void testCoverPage_showsSuccessfulTask() { - String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_SUCCESS).build()); - assertThat(content).contains("task-success"); - assertThat(content).doesNotContain("

FAILURE

"); - assertThat(content).contains("

SUCCESS

"); - assertThat(content).doesNotContain("

UP_TO_DATE

"); - } - - @Test - void testCoverPage_showsUpToDateTask() { - String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_UP_TO_DATE).build()); - assertThat(content).contains("task-up-to-date"); - assertThat(content).doesNotContain("

FAILURE

"); - assertThat(content).doesNotContain("

SUCCESS

"); - assertThat(content).contains("

UP_TO_DATE

"); - } - - @Test - void testCoverPage_failedAreFirst() { - String content = - getCoverPage( - EMPTY_PROJECT.toBuilder() - .addTask(EMPTY_TASK_UP_TO_DATE) - .addTask(EMPTY_TASK_FAILURE) - .addTask(EMPTY_TASK_SUCCESS) - .build()); - assertThat(content).contains("

FAILURE

"); - assertThat(content).contains("

SUCCESS

"); - assertThat(content).contains("

UP_TO_DATE

"); - assertThat(content).containsMatch("(?s)

FAILURE

.*

SUCCESS

"); - assertThat(content).containsMatch("(?s)

FAILURE

.*

UP_TO_DATE

"); - assertThat(content).doesNotContainMatch("(?s)

SUCCESS

.*

FAILURE

"); - assertThat(content).doesNotContainMatch("(?s)

UP_TO_DATE

.*

FAILURE

"); - } - - @Test - void testCoverPage_failingTask_statusIsFailure() { - String content = - getCoverPage( - EMPTY_PROJECT.toBuilder() - .addTask(EMPTY_TASK_UP_TO_DATE) - .addTask(EMPTY_TASK_FAILURE) - .addTask(EMPTY_TASK_SUCCESS) - .build()); - assertThat(content).contains("Failed: task-failure"); - } - - @Test - void testCoverPage_noFailingTask_statusIsSuccess() { - String content = - getCoverPage( - EMPTY_PROJECT.toBuilder() - .addTask(EMPTY_TASK_UP_TO_DATE) - .addTask(EMPTY_TASK_SUCCESS) - .build()); - assertThat(content).contains("Success!"); - } - - @Test - void testGetFilesToUpload_containsCssFile() { - ImmutableMap files = getGeneratedFiles(EMPTY_PROJECT); - assertThat(files).containsKey(filenameJoiner.join("css", "style.css")); - assertThat(files.get(filenameJoiner.join("css", "style.css"))).contains("body {"); - assertThat(files.get("index.html")) - .contains(""); - } - - @Test - void testCreateReportFiles_taskWithLog() { - ImmutableMap files = - getGeneratedFiles( - EMPTY_PROJECT.toBuilder() - .addTask( - EMPTY_TASK_SUCCESS.toBuilder() - .setUniqueName("my:name") - .setLog(toByteArraySupplier("my log data")) - .build()) - .build()); - assertThat(files).containsEntry(filenameJoiner.join("logs", "my-name.log"), "my log data"); - assertThat(files.get("index.html")).contains("[log]"); - } - - @Test - void testCreateReportFiles_taskWithoutLog() { - ImmutableMap files = - getGeneratedFiles( - EMPTY_PROJECT.toBuilder() - .addTask(EMPTY_TASK_SUCCESS.toBuilder().setUniqueName("my:name").build()) - .build()); - assertThat(files).doesNotContainKey("logs/my-name.log"); - assertThat(files.get("index.html")).contains("[log]"); - } - - @Test - void testCreateReportFiles_taskWithFilledReport() { - ImmutableMap files = - getGeneratedFiles( - EMPTY_PROJECT.toBuilder() - .addTask( - EMPTY_TASK_SUCCESS.toBuilder() - .putReport( - "someReport", - FilesWithEntryPoint.create( - ImmutableMap.of( - Paths.get("path", "report.txt"), - toByteArraySupplier("report content")), - Paths.get("path", "report.txt"))) - .build()) - .build()); - assertThat(files).containsEntry(filenameJoiner.join("path", "report.txt"), "report content"); - assertThat(files.get("index.html")).contains("[someReport]"); - } - - @Test - void testCreateReportFiles_taskWithEmptyReport() { - ImmutableMap files = - getGeneratedFiles( - EMPTY_PROJECT.toBuilder() - .addTask( - EMPTY_TASK_SUCCESS.toBuilder() - .putReport( - "someReport", - FilesWithEntryPoint.create( - ImmutableMap.of(), Paths.get("path", "report.txt"))) - .build()) - .build()); - assertThat(files).doesNotContainKey(filenameJoiner.join("path", "report.txt")); - assertThat(files.get("index.html")) - .contains("[someReport]"); - } - - @Test - void testCreateReportFiles_taskWithLogAndMultipleReports() { - ImmutableMap files = - getGeneratedFiles( - EMPTY_PROJECT.toBuilder() - .addTask( - EMPTY_TASK_SUCCESS.toBuilder() - .setUniqueName("my:name") - .setLog(toByteArraySupplier("log data")) - .putReport( - "filledReport", - FilesWithEntryPoint.create( - ImmutableMap.of( - Paths.get("path-filled", "report.txt"), - toByteArraySupplier("report content"), - Paths.get("path-filled", "other", "file.txt"), - toByteArraySupplier("some other content")), - Paths.get("path-filled", "report.txt"))) - .putReport( - "emptyReport", - FilesWithEntryPoint.create( - ImmutableMap.of(), Paths.get("path-empty", "report.txt"))) - .build()) - .build()); - assertThat(files) - .containsEntry(filenameJoiner.join("path-filled", "report.txt"), "report content"); - assertThat(files) - .containsEntry( - filenameJoiner.join("path-filled", "other", "file.txt"), "some other content"); - assertThat(files).containsEntry(filenameJoiner.join("logs", "my-name.log"), "log data"); - assertThat(files.get("index.html")) - .contains("[filledReport]"); - assertThat(files.get("index.html")).contains("[log]"); - assertThat(files.get("index.html")) - .contains("[emptyReport]"); - } -} diff --git a/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java b/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java deleted file mode 100644 index f420057ae..000000000 --- a/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gradle.plugin; - -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.truth.Truth.assertThat; -import static google.registry.gradle.plugin.GcsPluginUtils.getContentType; -import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint; -import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; -import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath; -import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs; -import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.nio.file.Files.createDirectories; -import static java.nio.file.Files.createDirectory; -import static java.nio.file.Files.createFile; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; - -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Optional; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** Tests for {@link GcsPluginUtilsTest} */ -final class GcsPluginUtilsTest { - - private static final Joiner filenameJoiner = Joiner.on(File.separator); - - @SuppressWarnings("WeakerAccess") - @TempDir - Path tmpDir; - - @Test - void testGetContentType_knownTypes() { - assertThat(getContentType("path/to/file.html")).isEqualTo("text/html"); - assertThat(getContentType("path/to/file.htm")).isEqualTo("text/html"); - assertThat(getContentType("path/to/file.log")).isEqualTo("text/plain"); - assertThat(getContentType("path/to/file.txt")).isEqualTo("text/plain"); - assertThat(getContentType("path/to/file.css")).isEqualTo("text/css"); - assertThat(getContentType("path/to/file.xml")).isEqualTo("text/xml"); - assertThat(getContentType("path/to/file.zip")).isEqualTo("application/zip"); - assertThat(getContentType("path/to/file.js")).isEqualTo("text/javascript"); - } - - @Test - void testGetContentType_unknownTypes() { - assertThat(getContentType("path/to/file.unknown")).isEqualTo("application/octet-stream"); - } - - @Test - void testUploadFileToGcs() { - Storage storage = mock(Storage.class); - uploadFileToGcs( - storage, "my-bucket", Paths.get("my", "filename.txt"), toByteArraySupplier("my data")); - verify(storage) - .create( - BlobInfo.newBuilder("my-bucket", "my/filename.txt") - .setContentType("text/plain") - .build(), - "my data".getBytes(UTF_8)); - verifyNoMoreInteractions(storage); - } - - @Test - void testUploadFilesToGcsMultithread() { - Storage storage = mock(Storage.class); - uploadFilesToGcsMultithread( - storage, - "my-bucket", - Paths.get("my", "folder"), - ImmutableMap.of( - Paths.get("some", "index.html"), toByteArraySupplier("some web page"), - Paths.get("some", "style.css"), toByteArraySupplier("some style"), - Paths.get("other", "index.html"), toByteArraySupplier("other web page"), - Paths.get("other", "style.css"), toByteArraySupplier("other style"))); - verify(storage) - .create( - BlobInfo.newBuilder("my-bucket", "my/folder/some/index.html") - .setContentType("text/html") - .build(), - "some web page".getBytes(UTF_8)); - verify(storage) - .create( - BlobInfo.newBuilder("my-bucket", "my/folder/some/style.css") - .setContentType("text/css") - .build(), - "some style".getBytes(UTF_8)); - verify(storage) - .create( - BlobInfo.newBuilder("my-bucket", "my/folder/other/index.html") - .setContentType("text/html") - .build(), - "other web page".getBytes(UTF_8)); - verify(storage) - .create( - BlobInfo.newBuilder("my-bucket", "my/folder/other/style.css") - .setContentType("text/css") - .build(), - "other style".getBytes(UTF_8)); - verifyNoMoreInteractions(storage); - } - - @Test - void testToByteArraySupplier_string() { - assertThat(toByteArraySupplier("my string").get()).isEqualTo("my string".getBytes(UTF_8)); - } - - @Test - void testToByteArraySupplier_stringSupplier() { - assertThat(toByteArraySupplier(() -> "my string").get()).isEqualTo("my string".getBytes(UTF_8)); - } - - @Test - void testToByteArraySupplier_file() throws Exception { - Path dir = createDirectory(tmpDir.resolve("arbitrary")); - Path file = createFile(dir.resolve("file.txt")); - Files.write(file, "some data".getBytes(UTF_8)); - assertThat(toByteArraySupplier(file.toFile()).get()).isEqualTo("some data".getBytes(UTF_8)); - } - - private ImmutableMap readAllFiles(FilesWithEntryPoint reportFiles) { - return reportFiles.files().entrySet().stream() - .collect( - toImmutableMap( - entry -> entry.getKey().toString(), - entry -> new String(entry.getValue().get(), UTF_8))); - } - - @Test - void testCreateReportFiles_destinationIsFile() throws Exception { - Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath()); - Path somePath = createDirectories(root.resolve("some/path")); - Path destination = createFile(somePath.resolve("file.txt")); - Files.write(destination, "some data".getBytes(UTF_8)); - // Since the entry point is obvious here - any hint given is just ignored. - File ignoredHint = createFile(root.resolve("ignored.txt")).toFile(); - - FilesWithEntryPoint files = - readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root); - - assertThat(files.entryPoint().toString()) - .isEqualTo(filenameJoiner.join("some", "path", "file.txt")); - assertThat(readAllFiles(files)) - .containsExactly(filenameJoiner.join("some", "path", "file.txt"), "some data"); - } - - @Test - void testCreateReportFiles_destinationDoesntExist() throws Exception { - Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath()); - File destination = root.resolve("non/existing.txt").toFile(); - assertThat(destination.isFile()).isFalse(); - assertThat(destination.isDirectory()).isFalse(); - // Since there are no files, any hint given is obviously wrong and will be ignored. - File ignoredHint = createFile(root.resolve("ignored.txt")).toFile(); - - FilesWithEntryPoint files = - readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); - - assertThat(files.entryPoint().toString()).isEqualTo(filenameJoiner.join("non", "existing.txt")); - assertThat(files.files()).isEmpty(); - } - - @Test - void testCreateReportFiles_noFiles() throws Exception { - Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath()); - Path destination = createDirectories(root.resolve("some/path")); - createDirectories(destination.resolve("a/b")); - createDirectory(destination.resolve("c")); - // Since there are not files, any hint given is obviously wrong and will be ignored. - File ignoredHint = createFile(root.resolve("ignored.txt")).toFile(); - - FilesWithEntryPoint files = - readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root); - - assertThat(files.entryPoint().toString()).isEqualTo(filenameJoiner.join("some", "path")); - assertThat(files.files()).isEmpty(); - } - - @Test - void testCreateReportFiles_oneFile() throws Exception { - Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath()); - Path destination = createDirectories(root.resolve("some/path")); - createDirectories(destination.resolve("a/b")); - createDirectory(destination.resolve("c")); - Files.write(createFile(destination.resolve("a/file.txt")), "some data".getBytes(UTF_8)); - // Since the entry point is obvious here - any hint given is just ignored. - File ignoredHint = createFile(root.resolve("ignored.txt")).toFile(); - - FilesWithEntryPoint files = - readFilesWithEntryPoint(destination.toFile(), Optional.of(ignoredHint), root); - - assertThat(files.entryPoint().toString()) - .isEqualTo(filenameJoiner.join("some", "path", "a", "file.txt")); - assertThat(readAllFiles(files)) - .containsExactly(filenameJoiner.join("some", "path", "a", "file.txt"), "some data"); - } - - /** - * Currently tests the "unimplemented" behavior. - * - *

TODO(guyben): switch to checking zip file instead. - */ - @Test - void testCreateReportFiles_multipleFiles_noHint() throws Exception { - Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath()); - Path destination = createDirectories(root.resolve("some/path")); - createDirectories(destination.resolve("a/b")); - createDirectory(destination.resolve("c")); - - Files.write(createFile(destination.resolve("index.html")), "some data".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8)); - - FilesWithEntryPoint files = - readFilesWithEntryPoint(destination.toFile(), Optional.empty(), root); - - assertThat(files.entryPoint().toString()) - .isEqualTo(filenameJoiner.join("some", "path", "path.zip")); - assertThat(readAllFiles(files).keySet()) - .containsExactly(filenameJoiner.join("some", "path", "path.zip")); - } - - /** - * Currently tests the "unimplemented" behavior. - * - *

TODO(guyben): switch to checking zip file instead. - */ - @Test - void testCreateReportFiles_multipleFiles_withBadHint() throws Exception { - Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath()); - Path destination = createDirectories(root.resolve("some/path")); - // This entry point points to a directory, which isn't an appropriate entry point - File badEntryPoint = createDirectories(destination.resolve("a/b")).toFile(); - createDirectory(destination.resolve("c")); - - Files.write(createFile(destination.resolve("index.html")), "some data".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8)); - - FilesWithEntryPoint files = - readFilesWithEntryPoint(destination.toFile(), Optional.of(badEntryPoint), root); - - assertThat(files.entryPoint().toString()) - .isEqualTo(filenameJoiner.join("some", "path", "path.zip")); - assertThat(readAllFiles(files).keySet()) - .containsExactly(filenameJoiner.join("some", "path", "path.zip")); - } - - @Test - void testCreateReportFiles_multipleFiles_withGoodHint() throws Exception { - Path root = toNormalizedPath(createDirectories(tmpDir.resolve("my/root")).toAbsolutePath()); - Path destination = createDirectories(root.resolve("some/path")); - createDirectories(destination.resolve("a/b")); - createDirectory(destination.resolve("c")); - // The hint is an actual file nested in the destination directory! - Path goodEntryPoint = createFile(destination.resolve("index.html")); - - Files.write(goodEntryPoint, "some data".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("a/index.html")), "wrong index".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("c/style.css")), "css file".getBytes(UTF_8)); - Files.write(createFile(destination.resolve("my_image.png")), "images".getBytes(UTF_8)); - - FilesWithEntryPoint files = - readFilesWithEntryPoint(destination.toFile(), Optional.of(goodEntryPoint.toFile()), root); - - assertThat(files.entryPoint().toString()) - .isEqualTo(filenameJoiner.join("some", "path", "index.html")); - assertThat(readAllFiles(files)) - .containsExactly( - filenameJoiner.join("some", "path", "index.html"), "some data", - filenameJoiner.join("some", "path", "a", "index.html"), "wrong index", - filenameJoiner.join("some", "path", "c", "style.css"), "css file", - filenameJoiner.join("some", "path", "my_image.png"), "images"); - } -} diff --git a/buildSrc/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/buildSrc/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 1f0955d45..000000000 --- a/buildSrc/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1 +0,0 @@ -mock-maker-inline diff --git a/config/nom_build.py b/config/nom_build.py index 2fc4f7d68..8894412d2 100644 --- a/config/nom_build.py +++ b/config/nom_build.py @@ -89,13 +89,6 @@ PROPERTIES = [ 'Allow connecting to plain HTTP repositories. This is provided ' 'to allow us to communicate to a local proxy when doing ' 'dependency updates.'), - Property('uploaderDestination', - 'Location to upload test reports to. Normally this should be a ' - 'GCS url (see also uploaderCredentialsFile)'), - Property('uploaderCredentialsFile', - 'json credentials file to use to upload test reports.'), - Property('uploaderMultithreadedUpload', - 'Whether to enable multithread upload.'), Property('verboseTestOutput', 'If true, show all test output in near-realtime.', 'false', diff --git a/dependencies.gradle b/dependencies.gradle index cd0b50686..1cff63330 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -53,13 +53,14 @@ ext { // TODO: Remove after the legacy console is deleted. 'com.google.closure-stylesheets:closure-stylesheets:1.5.0', 'com.google.javascript:closure-compiler:v20210505', - 'com.google.protobuf:protobuf-java:3.13.0', - 'com.google.template:soy:2021-02-01', 'org.seleniumhq.selenium:selenium-api:3.141.59', 'org.seleniumhq.selenium:selenium-chrome-driver:3.141.59', 'org.seleniumhq.selenium:selenium-java:3.141.59', 'org.seleniumhq.selenium:selenium-remote-driver:3.141.59', + // TODO: Migrate Soy Tofu to Soy Sauce (go/soysauce-migration) + 'com.google.protobuf:protobuf-java:3.13.0', + 'com.google.template:soy:2021-02-01', // CAPPED VERSIONS START HERE. diff --git a/gradle.properties b/gradle.properties index 55649a3ed..8a8b84822 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,9 +13,6 @@ org.gradle.parallel=true mavenUrl= pluginsUrl= allowInsecureProtocol= -uploaderDestination= -uploaderCredentialsFile= -uploaderMultithreadedUpload= verboseTestOutput=false enableDependencyLocking=true enableCrossReferencing=false diff --git a/prober/src/main/java/google/registry/monitoring/blackbox/handler/WebWhoisActionHandler.java b/prober/src/main/java/google/registry/monitoring/blackbox/handler/WebWhoisActionHandler.java index 3a79163a9..8566b0a2a 100644 --- a/prober/src/main/java/google/registry/monitoring/blackbox/handler/WebWhoisActionHandler.java +++ b/prober/src/main/java/google/registry/monitoring/blackbox/handler/WebWhoisActionHandler.java @@ -31,6 +31,8 @@ import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpResponseStatus; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import javax.inject.Inject; import org.joda.time.Duration; @@ -99,8 +101,8 @@ public class WebWhoisActionHandler extends ActionHandler { // Obtain url to be redirected to URL url; try { - url = new URL(response.headers().get("Location")); - } catch (MalformedURLException e) { + url = new URI(response.headers().get("Location")).toURL(); + } catch (MalformedURLException | URISyntaxException | IllegalArgumentException e) { // in case of error, log it, and let ActionHandler's exceptionThrown method deal with it throw new FailureException( "Redirected Location was invalid. Given Location was: " diff --git a/prober/src/test/java/google/registry/monitoring/blackbox/handler/WebWhoisActionHandlerTest.java b/prober/src/test/java/google/registry/monitoring/blackbox/handler/WebWhoisActionHandlerTest.java index 96b84fd95..1944d42a7 100644 --- a/prober/src/test/java/google/registry/monitoring/blackbox/handler/WebWhoisActionHandlerTest.java +++ b/prober/src/test/java/google/registry/monitoring/blackbox/handler/WebWhoisActionHandlerTest.java @@ -172,7 +172,7 @@ class WebWhoisActionHandlerTest { assertThat(future.isSuccess()).isFalse(); // Ensures that we fail as a result of a FailureException. - assertThat(future.cause() instanceof FailureException).isTrue(); + assertThat(future.cause()).isInstanceOf(FailureException.class); } @Test