From a456daa0b2f9930a8918f657e5a2e66702a768ed Mon Sep 17 00:00:00 2001 From: aram price Date: Thu, 13 Aug 2020 14:10:41 -0700 Subject: [PATCH 1/3] `./hack/module.sh` learns `codegen` command Runs code generation on a per-module basis. If `CONTAINED` is not set the code generation is run in a container. Mount point in docker is randomzied to simulate Concourse. Introduce K8S_PKG_VERSION to make room to build different versions eventually. --- hack/boilerplate.go.txt | 4 ++ hack/codegen.sh | 101 ++++++++++++++++++++++++++++++++++++++++ hack/module.sh | 7 ++- 3 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 hack/boilerplate.go.txt create mode 100755 hack/codegen.sh diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 000000000..422671ea8 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,4 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ diff --git a/hack/codegen.sh b/hack/codegen.sh new file mode 100755 index 000000000..147c505bc --- /dev/null +++ b/hack/codegen.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# Copyright 2020 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +GOPATH="${GOPATH:-$(mktemp -d)}" + +K8S_PKG_VERSION="${K8S_PKG_VERSION:-"1.19"}" # TODO: set this in k8s-code-generator-{} image +CODEGEN_IMAGE=${CODEGEN_IMAGE:-"gcr.io/tanzu-user-authentication/k8s-code-generator-${K8S_PKG_VERSION}:latest"} + +BASE_PKG="github.com/suzerain-io/placeholder-name" + +function codegen::ensure_module_in_gopath() { + local pkg_name="${MOD_DIR/#${ROOT}\//}" + local pkg_gosrc_path="${GOPATH}/src/${BASE_PKG}/${pkg_name}" + + if [[ ! -e "${pkg_gosrc_path}" ]]; then + mkdir -p "$(dirname "${pkg_gosrc_path}")" + ln -s "${ROOT}/${pkg_name}" "${pkg_gosrc_path}" + fi +} + +function codegen::invoke_code_generator() { + local generator_command="${1}" + local mod_basename_for_version="${2}" + shift 2 # generator args are now in $@ + + if [ "${BASH_VERSINFO[0]}" -lt 5 ]; then + echo "ERROR: invalid BASH version" + echo " using v${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}.${BASH_VERSINFO[2]} @ ${BASH}" + echo " require v5.0.0+" + exit 1 + fi + + bash "${GOPATH}/src/k8s.io/code-generator/${generator_command}.sh" \ + "$@" \ + --go-header-file "${ROOT}/hack/boilerplate.go.txt" | + sed "s|^|${mod_basename_for_version} > ${generator_command} > |" +} + +function codegen::generate_for_module() { + local mod_basename_for_version="${1}" + + case "${mod_basename_for_version}" in + 1.19/api) + codegen::invoke_code_generator generate-groups "${mod_basename_for_version}" \ + deepcopy,defaulter \ + "${BASE_PKG}/kubernetes/1.19/api/generated" \ + "${BASE_PKG}/kubernetes/1.19/api/apis" \ + "placeholder:v1alpha1 crdsplaceholder:v1alpha1" + codegen::invoke_code_generator generate-internal-groups "${mod_basename_for_version}" \ + deepcopy,defaulter,conversion,openapi \ + "${BASE_PKG}/kubernetes/1.19/api/generated" \ + "${BASE_PKG}/kubernetes/1.19/api/apis" \ + "${BASE_PKG}/kubernetes/1.19/api/apis" \ + "placeholder:v1alpha1 crdsplaceholder:v1alpha1" + ;; + 1.19/client-go) + codegen::invoke_code_generator generate-groups "${mod_basename_for_version}" \ + client,lister,informer \ + "${BASE_PKG}/kubernetes/1.19/client-go" \ + "${BASE_PKG}/kubernetes/1.19/api/apis" \ + "placeholder:v1alpha1 crdsplaceholder:v1alpha1" + ;; + esac +} + +function codegen::generate() { + codegen::ensure_module_in_gopath + codegen::generate_for_module "${K8S_PKG_VERSION}/$(basename "${MOD_DIR}")" +} + +function codegen::usage() { + echo "Error: must be specified" + echo " ${BASH_SOURCE[0]} [codegen::generate]" + exit 1 +} + +function codegen::main() { + local codegen_command="${1}" + + if [[ -n "${CONTAINED:-}" ]]; then + "${codegen_command}" + else + DOCKER_ROOT_DIR="/tmp/${RANDOM}/${BASE_PKG}" + DOCKER_MOD_DIR="${DOCKER_ROOT_DIR}/${MOD_DIR/#${ROOT}\//}" + + docker run --rm \ + --env CONTAINED=1 \ + --env MOD_DIR="${DOCKER_MOD_DIR}" \ + --volume "${ROOT}:${DOCKER_ROOT_DIR}" \ + --workdir "${DOCKER_MOD_DIR}" \ + "${CODEGEN_IMAGE}" \ + "${DOCKER_ROOT_DIR}/hack/$(basename "${BASH_SOURCE[0]}")" \ + "${codegen_command}" + fi +} + +codegen::main "${1:-"codegen::usage"}" diff --git a/hack/module.sh b/hack/module.sh index ebea9716c..e8b2ff51f 100755 --- a/hack/module.sh +++ b/hack/module.sh @@ -33,6 +33,10 @@ function unittest_cmd() { echo "${cmd} -short -race ./..." } +function codegen_cmd() { + echo "${ROOT}/hack/codegen.sh codegen::generate" +} + # The race detector is slow, so sometimes you don't want to use it function unittest_no_race_cmd() { if [ -x "$(command -v gotest)" ]; then @@ -62,7 +66,7 @@ function with_modules() { function usage() { echo "Error: must be specified" - echo " do.sh [tidy, lint, test, unittest, unittest_no_race]" + echo " do.sh [tidy, lint, test, unittest, unittest_no_race, codegen]" exit 1 } @@ -73,6 +77,7 @@ function main() { 'test') with_modules 'test_cmd' ;; 'unittest') with_modules 'unittest_cmd' ;; 'unittest_no_race') with_modules 'unittest_no_race_cmd' ;; + 'codegen') with_modules 'codegen_cmd' ;; *) usage ;; esac } From 7fa8f7797a747b49a357fa6fe2bb26f4c78d7f20 Mon Sep 17 00:00:00 2001 From: aram price Date: Fri, 14 Aug 2020 17:34:21 -0700 Subject: [PATCH 2/3] `hack/module.sh` learns `codegen_verify` --- hack/codegen.sh | 27 ++++++++++++++++++++++++++- hack/module.sh | 7 ++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/hack/codegen.sh b/hack/codegen.sh index 147c505bc..91e78db5d 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -68,8 +68,33 @@ function codegen::generate_for_module() { } function codegen::generate() { + local mod_basename_for_version + mod_basename_for_version="${K8S_PKG_VERSION}/$(basename "${MOD_DIR}")" + codegen::ensure_module_in_gopath - codegen::generate_for_module "${K8S_PKG_VERSION}/$(basename "${MOD_DIR}")" + codegen::generate_for_module "${mod_basename_for_version}" +} + +function codegen::verify() { + local have_stash='' + if [[ "$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ')" -ne "0" ]]; then + git stash --all >/dev/null 2>&1 && have_stash=1 + fi + + codegen::generate + + if [[ "$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ')" -eq "0" ]]; then + echo "Generated code in ${MOD_DIR} up to date." + else + echo "Generated code in ${MOD_DIR} is out of date." + echo "Please run hack/module.sh codegen" + git diff "${ROOT}" + git checkout "${ROOT}" + fi + + if [[ -n "${have_stash}" ]]; then + git stash pop >/dev/null 2>&1 + fi } function codegen::usage() { diff --git a/hack/module.sh b/hack/module.sh index e8b2ff51f..f5c948342 100755 --- a/hack/module.sh +++ b/hack/module.sh @@ -37,6 +37,10 @@ function codegen_cmd() { echo "${ROOT}/hack/codegen.sh codegen::generate" } +function codegen_verify_cmd() { + echo "${ROOT}/hack/codegen.sh codegen::verify" +} + # The race detector is slow, so sometimes you don't want to use it function unittest_no_race_cmd() { if [ -x "$(command -v gotest)" ]; then @@ -66,7 +70,7 @@ function with_modules() { function usage() { echo "Error: must be specified" - echo " do.sh [tidy, lint, test, unittest, unittest_no_race, codegen]" + echo " do.sh [tidy, lint, test, unittest, unittest_no_race, codegen, codegen_verify]" exit 1 } @@ -78,6 +82,7 @@ function main() { 'unittest') with_modules 'unittest_cmd' ;; 'unittest_no_race') with_modules 'unittest_no_race_cmd' ;; 'codegen') with_modules 'codegen_cmd' ;; + 'codegen_verify') with_modules 'codegen_verify_cmd' ;; *) usage ;; esac } From cedd47b92ee303dae93a2b3c5a5482cce9ac11db Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Tue, 18 Aug 2020 09:22:44 -0400 Subject: [PATCH 3/3] hack/codegen.sh: fix stashing, symlinking, failure, and usage Signed-off-by: Andrew Keesler --- hack/codegen.sh | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/hack/codegen.sh b/hack/codegen.sh index 91e78db5d..2981b97e6 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -13,8 +13,12 @@ CODEGEN_IMAGE=${CODEGEN_IMAGE:-"gcr.io/tanzu-user-authentication/k8s-code-genera BASE_PKG="github.com/suzerain-io/placeholder-name" function codegen::ensure_module_in_gopath() { - local pkg_name="${MOD_DIR/#${ROOT}\//}" - local pkg_gosrc_path="${GOPATH}/src/${BASE_PKG}/${pkg_name}" + # This should be something like "kubernetes/1.19/api". + local pkg_name="$(realpath "--relative-to=$ROOT" "$MOD_DIR")" + + # Use --canonicalize-missing to since pkg_name could end up as "." - this would + # lead to a pkg_gosrc_path like "foo/bar/bat/." which ln(1) (below) does not like. + local pkg_gosrc_path="$(realpath --canonicalize-missing "${GOPATH}/src/${BASE_PKG}/${pkg_name}")" if [[ ! -e "${pkg_gosrc_path}" ]]; then mkdir -p "$(dirname "${pkg_gosrc_path}")" @@ -78,11 +82,17 @@ function codegen::generate() { function codegen::verify() { local have_stash='' if [[ "$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ')" -ne "0" ]]; then + # git stash requires the user.email and user.name to be set. We set these at + # a global scope so they don't overwrite the .git/config in the mounted repo + # from the host. + git config --global user.email "codegen_verify@whatever.com" + git config --global user.name "Codegen Verify" git stash --all >/dev/null 2>&1 && have_stash=1 fi codegen::generate + failure=0 if [[ "$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ')" -eq "0" ]]; then echo "Generated code in ${MOD_DIR} up to date." else @@ -90,16 +100,21 @@ function codegen::verify() { echo "Please run hack/module.sh codegen" git diff "${ROOT}" git checkout "${ROOT}" + failure=1 fi if [[ -n "${have_stash}" ]]; then git stash pop >/dev/null 2>&1 fi + + if [[ "$failure" -eq 1 ]]; then + exit 1 + fi } function codegen::usage() { echo "Error: must be specified" - echo " ${BASH_SOURCE[0]} [codegen::generate]" + echo " ${BASH_SOURCE[0]} [codegen::generate, codegen::verify]" exit 1 } @@ -110,7 +125,7 @@ function codegen::main() { "${codegen_command}" else DOCKER_ROOT_DIR="/tmp/${RANDOM}/${BASE_PKG}" - DOCKER_MOD_DIR="${DOCKER_ROOT_DIR}/${MOD_DIR/#${ROOT}\//}" + DOCKER_MOD_DIR="${DOCKER_ROOT_DIR}/$(realpath "--relative-to=$ROOT" "$MOD_DIR")" docker run --rm \ --env CONTAINED=1 \