From 27d0c58319ee186a52bd83b978fde80ace445dc3 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 29 Feb 2024 09:47:00 -0800 Subject: [PATCH] Change codegen scripts to work with Kube 1.29 - Also add support for having comments in kube-versions.txt. - Update boilerplate copyright dates for generated code files. --- hack/boilerplate.go.txt | 2 +- hack/lib/update-codegen.sh | 85 +++++++++++++++++++++++++++++++++----- hack/update.sh | 18 +++++--- hack/verify.sh | 5 ++- 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index a3f9d8108..6920843af 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,2 +1,2 @@ -// Copyright 2020-2023 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2024 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 diff --git a/hack/lib/update-codegen.sh b/hack/lib/update-codegen.sh index 8be088473..e2f27f276 100755 --- a/hack/lib/update-codegen.sh +++ b/hack/lib/update-codegen.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright 2020-2023 the Pinniped contributors. All Rights Reserved. +# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -58,6 +58,14 @@ export KUBE_VERSION KUBE_MINOR_VERSION="$(echo "${KUBE_VERSION}" | cut -d"." -f1-2)" export KUBE_MINOR_VERSION +# Check if it is a recent version of Kube. +KUBE_MAJOR_NUMBER="$(echo "${KUBE_VERSION}" | cut -d"." -f1)" # (e.g. '1') +KUBE_MINOR_NUMBER="$(echo "${KUBE_VERSION}" | cut -d"." -f2)" # (e.g. '28') +KUBE_1_29_OR_NEWER="no" +if [[ "$KUBE_MAJOR_NUMBER" -gt "1" || ( "$KUBE_MAJOR_NUMBER" == "1" && "$KUBE_MINOR_NUMBER" -ge "29" ) ]]; then + KUBE_1_29_OR_NEWER="yes" +fi + # KUBE_MODULE_VERSION is just version of client libraries (e.g., 'v0.28.9-rc-0'). KUBE_MODULE_VERSION="v0.$(echo "${KUBE_VERSION}" | cut -d '.' -f 2-)" export KUBE_MODULE_VERSION @@ -77,13 +85,19 @@ find "${OUTPUT_DIR}" -type f -exec sed -i "s|GENERATED_PKG|generated/${KUBE_MINO find "${OUTPUT_DIR}" -type f -not -name '*.tmpl' -exec rm {} \; find "${OUTPUT_DIR}" -type f -name '*.tmpl' -exec bash -c 'mv "$0" "${0%.tmpl}"' {} \; +# Starting in Kube 1.29, we need to use language level 1.21. +GO_LANG_LEVEL="1.13" +if [[ "$KUBE_1_29_OR_NEWER" == "yes" ]]; then + GO_LANG_LEVEL="1.21" +fi + # Make the generated API code its own Go module. echo "generating ${OUTPUT_DIR}/apis/go.mod..." cat << EOF > "${OUTPUT_DIR}/apis/go.mod" -// This go.mod file is generated by ./hack/codegen.sh. +// This go.mod file is generated by ./hack/update.sh. module ${BASE_PKG}/generated/${KUBE_MINOR_VERSION}/apis -go 1.13 +go $GO_LANG_LEVEL require ( k8s.io/apimachinery ${KUBE_MODULE_VERSION} @@ -91,20 +105,57 @@ require ( ) EOF +# Create a Go file that declares the package or else we can't use "go mod tidy" in the client subdirectory. +# Without this, go mod tidy complains that this directory "but does not contain package". +# We will delete this at the end of the script. +cat << EOF > "${OUTPUT_DIR}/apis/placeholder.go" +package apis +EOF + +echo "Showing initial apis/go.mod..." +cat "${OUTPUT_DIR}/apis/go.mod" | sed "s|^|cat-go-mod > |" + # Generate a go.sum without changing the go.mod by running go mod download. echo "running go mod download in ${OUTPUT_DIR}/apis/go.mod to generate a go.sum file..." (cd "${OUTPUT_DIR}/apis" && go mod download all 2>&1 | sed "s|^|go-mod-download > |") +# Starting in Kube 1.29, the code generator complains that we need to run go mod tidy first. +if [[ "$KUBE_1_29_OR_NEWER" == "yes" ]]; then + echo "running go mod tidy in ${OUTPUT_DIR}/apis..." + (cd "${OUTPUT_DIR}/apis" && go mod tidy 2>&1 | sed "s|^|go-mod-tidy > |") + echo "Showing updated apis/go.mod..." + cat "${OUTPUT_DIR}/apis/go.mod" | sed "s|^|cat-updated-go-mod > |" +fi + # Make the generated client code its own Go module. echo "generating ${OUTPUT_DIR}/client/go.mod..." mkdir client mkdir client/concierge mkdir client/supervisor -cat << EOF > "./client/go.mod" -// This go.mod file is generated by ./hack/codegen.sh. + +# Create a tools.go just to prevent "go mod tidy" from deleting these packages from the go.mod as unused deps. +# We will delete this at the end of the script. +cat << EOF > "${OUTPUT_DIR}/client/tools.go" +//go:build tools + +package tools + +import ( + _ "k8s.io/api" + _ "k8s.io/apimachinery" + _ "k8s.io/client-go" + _ "${BASE_PKG}/generated/${KUBE_MINOR_VERSION}/apis" +) +EOF + +# Create client/go.mod. +cat << EOF > "${OUTPUT_DIR}/client/go.mod" +// This go.mod file is generated by ./hack/update.sh. module ${BASE_PKG}/generated/${KUBE_MINOR_VERSION}/client -go 1.13 +go $GO_LANG_LEVEL + +replace ${BASE_PKG}/generated/${KUBE_MINOR_VERSION}/apis => ../apis require ( k8s.io/api ${KUBE_MODULE_VERSION} @@ -112,14 +163,28 @@ require ( k8s.io/client-go ${KUBE_MODULE_VERSION} ${BASE_PKG}/generated/${KUBE_MINOR_VERSION}/apis v0.0.0 ) - -replace ${BASE_PKG}/generated/${KUBE_MINOR_VERSION}/apis => ../apis EOF +echo "Showing initial client/go.mod..." +cat "${OUTPUT_DIR}/client/go.mod" | sed "s|^|cat-go-mod > |" + # Generate a go.sum without changing the go.mod by running go mod download. echo "running go mod download in ${OUTPUT_DIR}/client/go.mod to generate a go.sum file..." (cd "${OUTPUT_DIR}/client" && go mod download all 2>&1 | sed "s|^|go-mod-download > |") +# Starting in Kube 1.29, the code generator complains that we need to run go mod tidy first. +if [[ "$KUBE_1_29_OR_NEWER" == "yes" ]]; then + echo "running go mod tidy in ${OUTPUT_DIR}/client..." + (cd "${OUTPUT_DIR}/client" && go mod tidy 2>&1 | sed "s|^|go-mod-tidy > |") + echo "Showing updated client/go.mod..." + cat "${OUTPUT_DIR}/client/go.mod" | sed "s|^|cat-updated-go-mod > |" +fi + +# These files were only created above to satisfy the above usages of "go mod tidy" in this script. +# We don't need to commit them. Delete them before the final "go mod tidy" so that unused +# imports can be deleted by the usages of "go mod tidy" below. +rm -f apis/placeholder.go client/tools.go + # Generate API-related code for our public API groups echo "generating API-related code for our public API groups..." chmod -R +x "${GOPATH}/src/k8s.io/code-generator/" @@ -158,7 +223,7 @@ echo "generating API-related code for our internal API groups..." --go-header-file "${ROOT}/hack/boilerplate.go.txt" -v "$debug_level" 2>&1 | sed "s|^|gen-supervisor-int-api > |" ) -# Tidy up the .../apis module +# Tidy the apis module after codegen. echo "tidying ${OUTPUT_DIR}/apis/go.mod..." (cd apis && go mod tidy 2>&1 | sed "s|^|go-mod-tidy > |") @@ -181,7 +246,7 @@ echo "generating client code for our public API groups..." --go-header-file "${ROOT}/hack/boilerplate.go.txt" -v "$debug_level" 2>&1 | sed "s|^|gen-supervisor-client > |" ) -# Tidy up the .../client module +# Tidy the client module after codegen. echo "tidying ${OUTPUT_DIR}/client/go.mod..." (cd client && go mod tidy 2>&1 | sed "s|^|go-mod-tidy > |") diff --git a/hack/update.sh b/hack/update.sh index 6b0cbca40..6f787c53b 100755 --- a/hack/update.sh +++ b/hack/update.sh @@ -1,22 +1,28 @@ #!/usr/bin/env bash -# Copyright 2020-2022 the Pinniped contributors. All Rights Reserved. +# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -# Generate code. -xargs -n 1 -P 8 "$ROOT/hack/lib/update-codegen.sh" < "${ROOT}/hack/lib/kube-versions.txt" +# Generate code, running each Kube version as a separate process, with up to 8 parallel processes at a time. +cat "${ROOT}/hack/lib/kube-versions.txt" | grep -v '^#' | xargs -n 1 -P 8 "$ROOT/hack/lib/update-codegen.sh" # Copy the latest version into a ./generated/latest directory so we can depend on it without nested modules. -LATEST_MINOR_VERSION="$(head -1 < "${ROOT}/hack/lib/kube-versions.txt" | cut -d"." -f1-2)" +LATEST_MINOR_VERSION="$(cat "${ROOT}/hack/lib/kube-versions.txt" | grep -v '^#' | cut -d"." -f1-2 | head -1)" LATEST_ROOT="$ROOT/generated/latest" rm -rf "$LATEST_ROOT" cp -r "$ROOT/generated/$LATEST_MINOR_VERSION/" "$LATEST_ROOT" -find "$LATEST_ROOT" \( -name "go.mod" -or -name "go.sum" -or -name "README.adoc" \) -delete + +# Delete the go.mod and go.sum because we do not want latest to be a nested module. +# Don't delete the README.adoc file so we can share GitHub URLs to it, like a permalink. +find "$LATEST_ROOT" \( -name "go.mod" -or -name "go.sum" \) -delete +# Delete the CRDs because latest is a go package, so it only needs the go files. rm -r "$LATEST_ROOT/crds" + +# Update the import statements in the latest package to make them refer to itself. if [[ "$(uname -s)" == "Linux" ]]; then # docker on linux preserves the root ownership of the output files of update-codegen.sh, # so chown the files before editing them. @@ -29,7 +35,7 @@ else find "$LATEST_ROOT" -type f -print0 | xargs -0 sed -i '' -e "s|go.pinniped.dev/generated/$LATEST_MINOR_VERSION|go.pinniped.dev/generated/latest|g" fi -# Copy each CRD yaml to the app which should cause it to be installed. +# Copy each generated CRD yaml to the app which should cause it to be installed. cp "$ROOT"/generated/"$LATEST_MINOR_VERSION"/crds/*.supervisor.*.yaml "$ROOT/deploy/supervisor" cp "$ROOT"/generated/"$LATEST_MINOR_VERSION"/crds/*.concierge.*.yaml "$ROOT/deploy/concierge" diff --git a/hack/verify.sh b/hack/verify.sh index bf94faadf..e6ca9cf48 100755 --- a/hack/verify.sh +++ b/hack/verify.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash -# Copyright 2020 the Pinniped contributors. All Rights Reserved. +# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -xargs "$ROOT/hack/lib/verify-codegen.sh" < "${ROOT}/hack/lib/kube-versions.txt" +cat "${ROOT}/hack/lib/kube-versions.txt" | grep -v '^#' | xargs "$ROOT/hack/lib/verify-codegen.sh" + "$ROOT/hack/module.sh" lint