Stop depending on GCS public access for Kokoro (#2907)

We used to publish test artifacts to a Maven repo on GCS, for use by
schema tests. For this to work with Kokoro, the GCS bucket must be
accessible to all users.

To comply with the no-public-user requirement, we store the necessary
jars at at well-known bucket and map them into Kokoro. This strategy
cannot be used on the Maven repo because only a small number of files
with fixed names may be mapped. With the Maven repo, there are too many
files to map.

config/nom_build.py

@@ -117,28 +117,19 @@ PROPERTIES = [
     Property('dbUser', 'Database user name for use in connection'),
     Property('dbPassword', 'Database password for use in connection'),
 
-    Property('publish_repo',
-             'Maven repository that hosts the Cloud SQL schema jar and the '
-             'registry server test jars. Such jars are needed for '
-             'server/schema integration tests. Please refer to <a '
-             'href="./integration/README.md">integration project</a> for more '
-             'information.'),
-    Property('baseSchemaTag',
-             'The nomulus version tag of the schema for use in the schema'
-             'deployment integration test (:db:schemaIncrementalDeployTest)'),
-    Property('schema_version',
-             'The nomulus version tag of the schema for use in a database'
-             'integration test.'),
-    Property('nomulus_version',
-             'The version of nomulus to test against in a database '
-             'integration test.'),
     Property('dot_path',
              'The path to "dot", part of the graphviz package that converts '
              'a BEAM pipeline to image. Setting this property to empty string '
              'will disable image generation.',
              '/usr/bin/dot'),
     Property('pipeline',
-             'The name of the Beam pipeline being staged.')
+             'The name of the Beam pipeline being staged.'),
+    Property('nomulus_env',
+             'For use by scripts. Normally not set manually.'),
+    Property('schema_env',
+             'For use by scripts. Normally not set manually.'),
+    Property('schemaTestArtifactsDir',
+             'For use by scripts. Normally not set manually.')
 ]
 
 GRADLE_FLAGS = [

core/build.gradle

@@ -646,23 +646,6 @@ artifacts {
   nomulus_test testUberJar
 }
 
-publishing {
-  repositories {
-    maven {
-      url project.publish_repo
-    }
-  }
-  publications {
-    nomulusTestsPublication(MavenPublication) {
-      groupId 'google.registry'
-      artifactId 'nomulus_test'
-      version project.nomulus_version
-      artifact nomulusFossJar
-      artifact testUberJar
-    }
-  }
-}
-
 task buildToolImage(dependsOn: nomulus, type: Exec) {
   commandLine 'docker', 'build', '-t', 'nomulus-tool', '.'
 }

db/build.gradle

@@ -126,22 +126,6 @@ artifacts {
   schema schemaJar
 }
 
-publishing {
-  repositories {
-    maven {
-      url project.publish_repo
-    }
-  }
-  publications {
-    sqlSchemaPublication(MavenPublication) {
-      groupId 'google.registry'
-      artifactId 'schema'
-      version project.schema_version
-      artifact schemaJar
-    }
-  }
-}
-
 // Adds flyway tasks such as: flywayInfo, flywayValidate, flywayMigrate (
 // deploying the schema in local repository), and flywayClean (dropping all data
 // in the database). The latter two commands are disallowed in environments
@@ -248,14 +232,10 @@ if (ext.isRestricted()) {
   }
 }
 
-if (project.baseSchemaTag != '') {
+if (project.schemaTestArtifactsDir != '') {
-  repositories {
-    maven {
-      url project.publish_repo
-    }
-  }
   dependencies {
-    integration "google.registry:schema:${project.baseSchemaTag}"
+    // For schemaIncrementalDeployTest, which only uses sandbox schema
+    integration files("${project.schemaTestArtifactsDir}/schema.sandbox.jar")
   }
 
   // Checks if Flyway scripts can be deployed to an existing database with

gradle.properties

@@ -22,9 +22,8 @@ dbServer=
 dbName=postgres
 dbUser=
 dbPassword=
-publish_repo=
-baseSchemaTag=
-schema_version=
-nomulus_version=
 dot_path=/usr/bin/dot
 pipeline=
+nomulus_env=
+schema_env=
+schemaTestArtifactsDir=

integration/README.md

@@ -1,50 +1,38 @@
 ## Summary
 
-This project runs cross-version server/schema integration tests with arbitrary
+This subproject provides two integration tests that ensure schema safety:
-version pairs. It may be used by presubmit tests and continuous-integration
-tests, or as a gating test during release and/or deployment.
 
-## Maven Dependencies
+*  One test checks for edits to Flyway scripts already deployed to Sandbox
+   Production. Such edits will cause schema deployment failure.
+*  Another test runs cross-version server/schema integration tests between a
+   pull request and the deployed release in Sandbox or Production. If a pull
+   request fails this test, it either contains schema changes not compatible
+   with Sandbox/Production binaries, or binaries not compatible with the
+   current schema in Sandbox/Production. This test may be include in presubmit
+   testing.
 
-This release process is expected to publish the following Maven dependencies to
+## Test Artifacts
-a well-known repository:
 
-*   google.registry:schema, which contains the schema DDL scripts. This is done
+To support the tests above, each release generates the following test artifacts:
-    by the ':db:publish' task.
-*   google.registry:nomulus_test, which contains the nomulus classes and
-    dependencies needed for the integration tests. This is done by the
-    ':core:publish' task.
 
-After each deployment in sandbox or production, the deployment process is
+*  schema.jar: The flyway scripts.
-expected to save the version tag of the binary or schema along with the
+*  nomulus-public.jar:  The open-source java classes.
-environment. These tags will be made available to test runners.
+*  nomulus-tests-alldeps.jar: Uber jar with schema test classes and all
+   third-party dependencies.
+
+After each deployment in sandbox or production, the deployment process copies
+these artifacts to a well-known location, and appends the environment tag to
+the file names.
 
 ## Usage
 
-The ':integration:sqlIntegrationTest' task is the test runner. It uses the
+Use the convenience scripts in the `integration` folder to run the tests. 
-following properties:
 
-*   nomulus_version: a Registry server release tag, or 'local' if the code in
+```bash
-    the local Git tree should be used.
+./integration/run_schema_check.sh -p domain-registry-dev
-*   schema_version: a schema release tag, or 'local' if the code in the local
-    Git tree should be used.
-*   publish_repo: the Maven repository where release jars may be found. This is
-    required if neither of the above is 'local'.
 
-Given a program 'fetch_version_tag' that retrieves the currently deployed
+./integration/run_compatibility_tests.sh -p domain-registry-dev -s sql
-version tag of SQL schema or server binary in a particular environment (which as
+./integration/run_compatibility_tests.sh -p domain-registry-dev -s nomulus
-mentioned earlier are saved by the deployment process), the following code
-snippet checks if the current PR or local clone has schema changes, and if yes,
-tests the production server's version with the new schema.
-
-```shell
-current_prod_schema=$(fetch_version_tag schema production)
-current_prod_server=$(fetch_version_tag server production)
-schema_changes=$(git diff ${current_prod_schema} --name-only \
-  ./db/src/main/resources/sql/flyway/ | wc -l)
-[[ schema_changes -gt  0 ]] && ./gradlew :integration:sqlIntegrationTest \
-  -Ppublish_repo=${REPO} -Pschema_version=local \
-  -Pnomulus_version=current_prod_server
 ```
 
 ## Implementation Notes

integration/build.gradle

@@ -18,22 +18,16 @@
 import static com.google.common.base.Preconditions.checkArgument
 import static com.google.common.base.Strings.isNullOrEmpty
 
-if (schema_version == '' || nomulus_version == '') {
+if (schema_env == '' || nomulus_env == '') {
   return
 }
 
 def USE_LOCAL = 'local'
 
-if (schema_version != USE_LOCAL || nomulus_version != USE_LOCAL) {
+if (schema_env != USE_LOCAL || nomulus_env != USE_LOCAL) {
   checkArgument(
-      !isNullOrEmpty(publish_repo),
+      !isNullOrEmpty(schemaTestArtifactsDir),
-      'The publish_repo is required when remote jars are needed.')
+      'The schemaTestArtifactsDir is required when deployed jars are needed.')
-
-  repositories {
-    maven {
-      url project.publish_repo
-    }
-  }
 }
 
 def testUberJarName = ''
@@ -41,25 +35,25 @@ def testUberJarName = ''
 // Might need to add this back if we re-add nebula-lint
 // gradleLint.ignore('unused-dependency') {
 dependencies {
-  if (schema_version == USE_LOCAL) {
+  if (schema_env == USE_LOCAL) {
     testRuntimeOnly project(path: ':db', configuration: 'schema')
   } else {
-    testRuntimeOnly "google.registry:schema:${schema_version}"
+    testRuntimeOnly files("${project.schemaTestArtifactsDir}/schema.${schema_env}.jar")
   }
-  if (nomulus_version == USE_LOCAL) {
+  if (nomulus_env == USE_LOCAL) {
     testRuntimeOnly project(path: ':core', configuration: 'nomulus_test')
     testUberJarName = 'nomulus-tests-alldeps.jar'
   } else {
-    testRuntimeOnly "google.registry:nomulus_test:${nomulus_version}:public"
+    testRuntimeOnly files("${project.schemaTestArtifactsDir}/nomulus-public.${nomulus_env}.jar")
-    testRuntimeOnly "google.registry:nomulus_test:${nomulus_version}:alldeps"
+    testRuntimeOnly files("${project.schemaTestArtifactsDir}/nomulus-tests-alldeps.${nomulus_env}.jar")
-    testUberJarName = "nomulus_test-${nomulus_version}-alldeps.jar"
+    testUberJarName = "nomulus-tests-alldeps.${nomulus_env}.jar"
   }
 }
 // }
 
 configurations.testRuntimeOnly.transitive = false
 
-def unpackedTestDir = "${projectDir}/build/unpackedTests/${nomulus_version}"
+def unpackedTestDir = "${projectDir}/build/unpackedTests/${nomulus_env}"
 
 // Extracts SqlIntegrationTestSuite.class to a temp folder. Gradle's test
 // runner only looks for runnable tests on a regular file system. However,
@@ -78,7 +72,7 @@ task extractSqlIntegrationTestSuite (type: Copy) {
   into unpackedTestDir
   includeEmptyDirs = false
 
-  if (nomulus_version == USE_LOCAL) {
+  if (nomulus_env == USE_LOCAL) {
     dependsOn ':core:testUberJar'
   }
 }

integration/run_compatibility_tests.sh

@@ -56,37 +56,50 @@ function runTest() {
   local deployed_system=${1}
   local version=${2}
   local dev_project=${3}
+  local env=${4}
 
-  local changes=$(getChangeCountSinceVersion ${deployed_system} ${version})
+#  local changes=$(getChangeCountSinceVersion ${deployed_system} ${version})
-  if [[ ${changes} = 0 ]]; then
+#  if [[ ${changes} = 0 ]]; then
-    echo "No relevant changes in ${deployed_system} since ${version}"
+#    echo "No relevant changes in ${deployed_system} since ${version}"
-    return 0
+#    return 0
+#  fi
+#
+#  echo "Found relevant changes in ${deployed_system} since ${version}"
+
+  if [[ -n "${SCHEMA_TEST_ARTIFACTS_DIR}" ]]; then
+    echo "Using schema test jars downloaded to ${SCHEMA_TEST_ARTIFACTS_DIR}"
+  else
+    SCHEMA_TEST_ARTIFACTS_DIR=$(mktemp -d)
+    echo "Created working dir ${SCHEMA_TEST_ARTIFACTS_DIR} for downloaded test jars."
+    trap 'rm -rf ${SCHEMA_TEST_ARTIFACTS_DIR}' EXIT
+    gcloud storage cp --verbosity=none \
+        "gs://${DEV_PROJECT}-deployed-tags/schema-test-artifacts/*.jar" \
+        "${SCHEMA_TEST_ARTIFACTS_DIR}"
   fi
 
-  echo "Found relevant changes in ${deployed_system} since ${version}"
+  local nomulus_env
-
+  local schema_env
-  local nomulus_version
-  local schema_version
 
   if [[ ${deployed_system} = "sql" ]]; then
-    schema_version=${version}
+    schema_env=${env}
-    nomulus_version="local"
+    nomulus_env="local"
   else
-    nomulus_version=${version}
+    nomulus_env=${env}
-    schema_version="local"
+    schema_env="local"
   fi
 
-  echo "Running test with -Pnomulus_version=${nomulus_version}" \
+  echo "Running test with -Pnomulus_env=${nomulus_env}" \
-      "-Pschema_version=${schema_version}"
+      "-Pschema_env=${schema_env}" \
+      "-PschemaTestArtifactsDir=${SCHEMA_TEST_ARTIFACTS_DIR}" \
 
   # The https scheme in the Maven repo URL below is required for Kokoro. See
   # ./run_schema_check.sh for more information.
   (cd ${SCRIPT_DIR}/..; \
       ./gradlew :integration:sqlIntegrationTest \
           -PdevProject=${dev_project} \
-          -Pnomulus_version=${nomulus_version} \
+          -Pnomulus_env=${nomulus_env} \
-          -Pschema_version=${schema_version} \
+          -Pschema_env=${schema_env} \
-          -Ppublish_repo=https://storage.googleapis.com/${dev_project}-deployed-tags/maven)
+          -PschemaTestArtifactsDir=${SCHEMA_TEST_ARTIFACTS_DIR})
 }
 
 set -e
@@ -128,19 +141,16 @@ if [[ -z "${ENV}" ]]; then
   SANDBOX_VERSION=$(fetchVersion ${DEPLOYED_SYSTEM} sandbox ${DEV_PROJECT})
   PROD_VERSION=$(fetchVersion ${DEPLOYED_SYSTEM} production ${DEV_PROJECT})
   if [[ ${SANDBOX_VERSION} = ${PROD_VERSION} ]]; then
-    VERSIONS=(${PROD_VERSION})
     echo "- sandbox and production at ${PROD_VERSION}"
+    runTest ${DEPLOYED_SYSTEM} ${SANDBOX_VERSION} ${DEV_PROJECT} sandbox
   else
-    VERSIONS=(${PROD_VERSION} ${SANDBOX_VERSION})
     echo "- sandbox at ${SANDBOX_VERSION}"
+    runTest ${DEPLOYED_SYSTEM} ${SANDBOX_VERSION} ${DEV_PROJECT} sandbox
     echo "- production at ${PROD_VERSION}"
+    runTest ${DEPLOYED_SYSTEM} ${PROD_VERSION} ${DEV_PROJECT} production
   fi
 else
   TARGET_VERSION=$(fetchVersion ${DEPLOYED_SYSTEM} ${ENV} ${DEV_PROJECT})
-  VERSIONS=(${TARGET_VERSION})
   echo "- ${ENV} at ${TARGET_VERSION}"
+  runTest ${DEPLOYED_SYSTEM} ${TARGET_VERSION} ${DEV_PROJECT} ${ENV}
 fi
-
-for v in "${VERSIONS[@]}"; do
-  runTest ${DEPLOYED_SYSTEM} ${v} ${DEV_PROJECT}
-done

integration/run_schema_check.sh

@@ -67,13 +67,17 @@ fi
 sandbox_tag=$(fetchVersion sql sandbox ${DEV_PROJECT})
 echo "Checking Flyway scripts against schema in Sandbox (${sandbox_tag})."
 
-# The URL of the Maven repo on GCS for the publish_repo parameter must use the
+if [[ -n "${SCHEMA_TEST_ARTIFACTS_DIR}" ]]; then
-# https scheme (https://storage.googleapis.com/{BUCKET}/{PATH}) in order to work
+  echo "Using schema test jars downloaded to ${SCHEMA_TEST_ARTIFACTS_DIR}"
-# with Kokoro. Gradle's alternative gcs scheme does not work on Kokoro: a GCP
+else
-# credential with proper scopes for GCS access is required even for public
+  SCHEMA_TEST_ARTIFACTS_DIR=$(mktemp -d)
-# buckets, however, Kokoro VM instances are not set up with such credentials.
+  echo "Created working dir ${SCHEMA_TEST_ARTIFACTS_DIR} for downloaded test jars."
-# Incidentally, gcs can be used on Cloud Build.
+  trap 'rm -rf ${SCHEMA_TEST_ARTIFACTS_DIR}' EXIT
+  gcloud storage cp --verbosity=none \
+      "gs://${DEV_PROJECT}-deployed-tags/schema-test-artifacts/*.jar" \
+      "${SCHEMA_TEST_ARTIFACTS_DIR}"
+fi
+
 (cd ${SCRIPT_DIR}/..; \
     ./gradlew :db:schemaIncrementalDeployTest \
-        -PbaseSchemaTag=${sandbox_tag} \
+        -PschemaTestArtifactsDir="${SCHEMA_TEST_ARTIFACTS_DIR}")
-        -Ppublish_repo=https://storage.googleapis.com/${DEV_PROJECT}-deployed-tags/maven)

release/cloudbuild-nomulus.yaml

@@ -154,8 +154,7 @@ steps:
       google/registry/beam/resave_all_epp_resources_pipeline_metadata.json \
       google.registry.beam.wipeout.WipeOutContactHistoryPiiPipeline \
       google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json
-# Tentatively build and publish Cloud SQL schema jar here, before the schema release process is
+# Build and upload the schema jar as well as other artifacts needed by the schema tests.
-# finalized. Also publish nomulus:core jars that are needed for server/schema compatibility tests.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
   # Set home for Gradle caches. Must be consistent with previous steps above
@@ -165,20 +164,8 @@ steps:
   - -c
   - |
     set -e
-    ./gradlew \
+    ./gradlew :db:schemaJar
-      :db:publish \
+    ./gradlew :core:nomulusFossJar :core:testUberJar
-      -PmavenUrl=gcs://domain-registry-maven-repository/maven \
-      -PpluginsUrl=gcs://domain-registry-maven-repository/plugins \
-      -Ppublish_repo=gcs://${PROJECT_ID}-deployed-tags/maven \
-      -Pschema_version=${TAG_NAME}
-    ./gradlew \
-      :core:publish \
-      -PmavenUrl=gcs://domain-registry-maven-repository/maven \
-      -PpluginsUrl=gcs://domain-registry-maven-repository/plugins \
-      -Ppublish_repo=gcs://${PROJECT_ID}-deployed-tags/maven \
-      -Pnomulus_version=${TAG_NAME}
-    # Upload jars for use by schema deployment and schema compatibility test.
-    # The jars published to GCS will be unaccessible to Kokoro once public access is removed.
     cp db/build/libs/schema.jar output/
     cp core/build/libs/nomulus-public.jar output/
     cp core/build/libs/nomulus-tests-alldeps.jar output/