initial commit on ci branch: migrates code from private repo

This commit is contained in:
Ryan Richard
2024-10-29 13:04:30 -07:00
parent a3d64bef62
commit 11bd69cf2d
194 changed files with 16873 additions and 16 deletions
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
repo=vmware-tanzu/pinniped
current_branch_name=$(git rev-parse --abbrev-ref HEAD)
if [[ "$current_branch_name" != "ci" ]]; then
echo "error: this script should only be used on the ci branch"
exit 1
fi
# Print the list of PRs to the screen.
PAGER='' gh pr list --base ci --repo $repo --limit 1000
# Exit if there are no PRs found.
count_prs=$(gh pr list --base ci --repo $repo --jq ". | length" --json "number")
if [[ "${count_prs}" == "0" ]]; then
exit 0
fi
read -p "Do you wish to approve and merge these PRs for the ci branch? y/n: " yn
case $yn in
[Yy]* );;
* ) exit 0;;
esac
gh pr list --base ci --repo $repo --json="number" --jq ".[] | .number" \
| xargs -I{} gh pr review {} --approve
gh pr list --base ci --repo $repo --json="number" --jq ".[] | .number" \
| xargs -I{} gh pr merge {} --merge --delete-branch
echo "now pulling the merged commits"
git pull --rebase --autostash
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
if ! [ -x "$(command -v gcloud)" ]; then
echo 'Error: Google Cloud SDK (gcloud) is not installed (see https://cloud.google.com/sdk/docs/quickstarts).' >&2
exit 1
fi
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
# Create (or recreate) a GKE acceptance cluster.
# Pro tip: The GCP Console UI can help you build this command.
# The following fields were customized, and all of the others are left as the GCP Console's defaults:
# - Cluster name
# - Cluster version - newest at the time
# - Num nodes - sized smaller to be cheaper
# - Maintenance window start and recurrence - to avoid downtime during business hours
# - Issue client certificate - to make it possible to use an admin kubeconfig without the GKE auth plugin
gcloud container --project "$PINNIPED_GCP_PROJECT" clusters create "gke-acceptance-cluster" \
--zone "us-central1-c" --no-enable-basic-auth --cluster-version "1.30.4-gke.1348000" --release-channel "regular" \
--machine-type "e2-medium" \
--image-type "COS_CONTAINERD" --disk-type "pd-balanced" --disk-size "100" --metadata disable-legacy-endpoints=true \
--scopes "https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append" \
--num-nodes "1" \
--logging=SYSTEM,WORKLOAD --monitoring=SYSTEM,STORAGE,POD,DEPLOYMENT,STATEFULSET,DAEMONSET,HPA,CADVISOR,KUBELET \
--enable-ip-alias \
--network "projects/$PINNIPED_GCP_PROJECT/global/networks/default" \
--subnetwork "projects/$PINNIPED_GCP_PROJECT/regions/us-central1/subnetworks/default" \
--no-enable-intra-node-visibility \
--default-max-pods-per-node "110" \
--security-posture=standard --workload-vulnerability-scanning=disabled --no-enable-master-authorized-networks \
--addons HorizontalPodAutoscaling,HttpLoadBalancing,GcePersistentDiskCsiDriver \
--enable-autoupgrade --enable-autorepair --max-surge-upgrade 1 --max-unavailable-upgrade 0 \
--binauthz-evaluation-mode=DISABLED --enable-managed-prometheus --enable-shielded-nodes --node-locations "us-central1-c" \
--maintenance-window-start "2020-07-01T03:00:00Z" --maintenance-window-end "2020-07-01T11:00:00Z" \
--maintenance-window-recurrence "FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR,SA,SU" \
--issue-client-certificate
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -e
if [ -z "$1" ]; then
echo "usage: $0 SECRET_NAME"
exit 1
fi
set -u
if ! command -v yq &> /dev/null; then
echo "Please install the yq CLI"
exit 1
fi
if ! command -v delta &> /dev/null; then
echo "Please install the delta CLI (brew install git-delta)"
exit 1
fi
if ! command -v gcloud &> /dev/null; then
echo "Please install the gcloud CLI"
exit 1
fi
if [[ -z "$(gcloud config list account --format "value(core.account)")" ]]; then
echo "Please run \`gcloud auth login\`"
exit 1
fi
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
# Create a temporary directory for secrets, cleaned up at the end of this script.
trap 'rm -rf "$TEMP_DIR"' EXIT
TEMP_DIR=$(mktemp -d) || exit 1
# Grab the current version.
echo "Downloading the latest version of '$1'..."
gcloud secrets versions access latest --secret="$1" --project "$PINNIPED_GCP_PROJECT" > "$TEMP_DIR/$1.yaml"
# Use yq to format the YAML into a consistent style.
# TODO: there is a bug in yq that strips leading comments on the first lines of a file when -P is used.
# For now, we'll skip the pretty-printing.
# yq eval -i -P '.' "$TEMP_DIR/$1.yaml"
yq eval -i '.' "$TEMP_DIR/$1.yaml"
cp "$TEMP_DIR/$1.yaml" "$TEMP_DIR/$1-original.yaml"
# Invoke $EDITOR to modify the file.
${EDITOR:-vim} "$TEMP_DIR/$1.yaml"
# Format the output from the editor just as we did before the edit.
# TODO: there is a bug in yq that strips leading comments on the first lines of a file when -P is used.
# For now, we'll skip the pretty-printing.
# yq eval -i -P '.' "$TEMP_DIR/$1.yaml"
yq eval -i '.' "$TEMP_DIR/$1.yaml"
# Dump the diff using git-delta.
( cd "$TEMP_DIR" && delta "$1-original.yaml" "$1.yaml" || true )
read -p "Save as new version of '$1' [yN]: " -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]
then
gcloud secrets versions add "$1" --data-file "$TEMP_DIR/$1.yaml" --project "$PINNIPED_GCP_PROJECT"
fi
+58
View File
@@ -0,0 +1,58 @@
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Some global fly config.
#
export FLY_CLI=/usr/local/bin/fly
export CONCOURSE_URL=https://ci.pinniped.dev
export CONCOURSE_TEAM=main
export CONCOURSE_TARGET=pinniped
export ROOT_DIR
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
#
# Some helper functions for the update-pipeline scripts to use.
#
function set_pipeline() {
# Ensure that fly is installed/upgraded/configured.
"$ROOT_DIR"/hack/setup-fly.sh
# Ensure that the user is authenticated with gcloud.
if [[ -z "$(gcloud config list account --format "value(core.account)")" ]]; then
echo "Please run \`gcloud auth login\` and try again."
exit 1
fi
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
# Local vars.
local pipeline_name=$1
local pipeline_file=$2
local gcloud_project="$PINNIPED_GCP_PROJECT"
local gcloud_secret_name=concourse-secrets
# Create/update the pipeline.
$FLY_CLI --target "$CONCOURSE_TARGET" set-pipeline \
--pipeline "$pipeline_name" \
--config "$pipeline_file" \
--load-vars-from <(gcloud secrets versions access latest \
--secret="$gcloud_secret_name" \
--project "$gcloud_project")
}
function ensure_time_resource_has_at_least_one_version() {
local pipeline_name=$1
local resource_name=$2
# Force the specified time resource to have at least one version. Idempotent.
# For a new pipeline, a time resource will have no versions until the specified time has occurred.
# For example, a once-per-night time resource will have no versions until that time
# has passed on the first night.
$FLY_CLI --target "$CONCOURSE_TARGET" check-resource \
--resource "$pipeline_name/$resource_name" \
--from "time:2000-01-01T00:00:00Z" >/dev/null
}
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
# To be run before local integration tests.
# From the pinniped repo:
# hack/prepare-for-integration-tests.sh --get-active-directory-vars "../pinniped-ci-branch/hack/get-aws-ad-env-vars.sh"
if [[ -z "$(gcloud config list account --format "value(core.account)")" ]]; then
echo "Please run \`gcloud auth login\`"
exit 1
fi
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
function _get_concourse_secret {
gcloud secrets versions access latest --secret="concourse-secrets" --project "$PINNIPED_GCP_PROJECT" | yq e "$1"
}
export PINNIPED_TEST_AD_HOST="$(_get_concourse_secret '.aws-ad-host')"
export PINNIPED_TEST_AD_DOMAIN="$(_get_concourse_secret '.aws-ad-domain')"
export PINNIPED_TEST_AD_BIND_ACCOUNT_USERNAME="$(_get_concourse_secret '.aws-ad-bind-account-username')"
export PINNIPED_TEST_AD_BIND_ACCOUNT_PASSWORD="$(_get_concourse_secret '.aws-ad-bind-account-password')"
export PINNIPED_TEST_AD_USER_UNIQUE_ID_ATTRIBUTE_NAME="objectGUID"
export PINNIPED_TEST_AD_USER_UNIQUE_ID_ATTRIBUTE_VALUE="$(_get_concourse_secret '.aws-ad-user-unique-id-attribute-value')"
export PINNIPED_TEST_AD_USER_USER_PRINCIPAL_NAME="$(_get_concourse_secret '.aws-ad-user-userprincipalname')"
export PINNIPED_TEST_AD_USER_PASSWORD="$(_get_concourse_secret '.aws-ad-user-password')"
export PINNIPED_TEST_AD_LDAPS_CA_BUNDLE="$(_get_concourse_secret '.aws-ad-ca-data')"
export PINNIPED_TEST_AD_USER_EXPECTED_GROUPS_DN="$(_get_concourse_secret '.aws-ad-expected-direct-groups-dn')"
export PINNIPED_TEST_AD_USER_EXPECTED_GROUPS_CN="$(_get_concourse_secret '.aws-ad-expected-direct-groups-cn')"
export PINNIPED_TEST_AD_USER_EXPECTED_GROUPS_SAMACCOUNTNAME="$(_get_concourse_secret '.aws-ad-expected-direct-and-nested-groups-samaccountnames')"
export PINNIPED_TEST_AD_USER_EXPECTED_GROUPS_SAMACCOUNTNAME_DOMAINNAMES="$(_get_concourse_secret '.aws-ad-expected-direct-and-nested-groups-samaccountname-domainnames')"
export PINNIPED_TEST_DEACTIVATED_AD_USER_SAMACCOUNTNAME="$(_get_concourse_secret '.aws-ad-deactivated-user-samaccountname')"
export PINNIPED_TEST_DEACTIVATED_AD_USER_PASSWORD="$(_get_concourse_secret '.aws-ad-deactivated-user-password')"
export PINNIPED_TEST_AD_USER_EMAIL_ATTRIBUTE_NAME="mail"
export PINNIPED_TEST_AD_USER_EMAIL_ATTRIBUTE_VALUE="$(_get_concourse_secret '.aws-ad-user-email-attribute-value')"
export PINNIPED_TEST_AD_DEFAULTNAMINGCONTEXT_DN="$(_get_concourse_secret '.aws-ad-defaultnamingcontext')"
export PINNIPED_TEST_AD_USERS_DN="$(_get_concourse_secret '.aws-ad-users-dn')"
unset -f _get_concourse_secret
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
# To be run before local integration tests.
# From the pinniped repo:
# hack/prepare-for-integration-tests.sh --get-github-vars "../pinniped-ci-branch/hack/get-github-env-vars.sh"
if [[ -z "$(gcloud config list account --format "value(core.account)")" ]]; then
echo "Please run \`gcloud auth login\`"
exit 1
fi
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
function _get_concourse_secret {
gcloud secrets versions access latest --secret="concourse-secrets" --project "$PINNIPED_GCP_PROJECT" | yq e "$1"
}
export PINNIPED_TEST_GITHUB_APP_CLIENT_ID="$(_get_concourse_secret '.github-app-client-id')"
export PINNIPED_TEST_GITHUB_APP_CLIENT_SECRET="$(_get_concourse_secret '.github-app-client-secret')"
export PINNIPED_TEST_GITHUB_OAUTH_APP_CLIENT_ID="$(_get_concourse_secret '.github-oauth-app-client-id')"
export PINNIPED_TEST_GITHUB_OAUTH_APP_CLIENT_SECRET="$(_get_concourse_secret '.github-oauth-app-client-secret')"
export PINNIPED_TEST_GITHUB_OAUTH_APP_ALLOWED_CALLBACK_URL="$(_get_concourse_secret '.github-oauth-app-allowed-callback-url')"
export PINNIPED_TEST_GITHUB_USER_USERNAME="$(_get_concourse_secret '.github-username')"
export PINNIPED_TEST_GITHUB_USER_PASSWORD="$(_get_concourse_secret '.github-password')"
export PINNIPED_TEST_GITHUB_USER_OTP_SECRET="$(_get_concourse_secret '.github-user-otp-secret')"
export PINNIPED_TEST_GITHUB_USERID="$(_get_concourse_secret '.github-userid')"
export PINNIPED_TEST_GITHUB_ORG="$(_get_concourse_secret '.github-org')"
export PINNIPED_TEST_GITHUB_EXPECTED_TEAM_NAMES="$(_get_concourse_secret '.github-expected-team-names')"
export PINNIPED_TEST_GITHUB_EXPECTED_TEAM_SLUGS="$(_get_concourse_secret '.github-expected-team-slugs')"
unset -f _get_concourse_secret
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Define some env vars
source "$script_dir/fly-helpers.sh"
# Setup and login if needed
"$ROOT_DIR"/hack/setup-fly.sh
# List all jobs that are currently running in CI.
# An empty result means that there are no jobs running.
for p in $($FLY_CLI --target "$CONCOURSE_TARGET" pipelines --json | jq -r ".[].name"); do
$FLY_CLI --target "$CONCOURSE_TARGET" jobs -p "$p" --json | jq -r ".[] | select(.next_build.status == \"started\") | (\"$p/\" + .name)"
done
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
pinniped_ci_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
pinniped_path="${1-$PWD}"
pinniped_ci_path="${2-$pinniped_ci_root}"
cd "$pinniped_path" || exit 1
if [[ ! -f "./hack/module.sh" ]]; then
echo "$pinniped_path does not appear to be the path to the source code repo directory"
exit 1
fi
if [[ ! -f "$pinniped_ci_path/hack/run-integration-tests.sh" ]]; then
echo "$pinniped_ci_path does not appear to be the path to the ci repo directory"
exit 1
fi
echo
echo "Running linters..."
./hack/module.sh lint
echo
echo "Running units..."
./hack/module.sh unittest
echo
echo "Running integrations..."
"$pinniped_ci_path"/hack/run-integration-tests.sh --from-clean-cluster
echo
echo "ALL TESTS PASSED"
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
# This script can be used to prepare a kind cluster and deploy the app
# in preparation for running the uninstall test.
# It will also output instructions on how to run the uninstall test.
set -euo pipefail
help=no
skip_build=no
pinniped_ci_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PARAMS=""
while (("$#")); do
case "$1" in
-h | --help)
help=yes
shift
;;
-s | --skip-build)
skip_build=yes
shift
;;
-*)
echo "Error: Unsupported flag $1" >&2
exit 1
;;
*)
PARAMS="$PARAMS $1"
shift
;;
esac
done
eval set -- "$PARAMS"
if [[ "$help" == "yes" ]]; then
me="$(basename "${BASH_SOURCE[0]}")"
echo "Usage:"
echo " $me [flags] [path/to/pinniped] [path/to/pinniped-ci-branch]"
echo
echo " path/to/pinniped default: \$PWD ($PWD)"
echo " path/to/pinniped-ci-branch default: the parent directory of this script ($pinniped_ci_root)"
echo
echo "Flags:"
echo " -h, --help: print this usage"
echo " -s, --skip-build: reuse the most recently built image of the app instead of building"
exit 1
fi
pinniped_path="${1-$PWD}"
pinniped_ci_path="${2-$pinniped_ci_root}"
if ! command -v kind >/dev/null; then
echo "Please install kind. e.g. 'brew install kind' for MacOS"
exit 1
fi
if ! command -v ytt >/dev/null; then
log_error "Please install ytt. e.g. 'brew tap k14s/tap && brew install ytt' for MacOS"
exit 1
fi
if ! command -v kapp >/dev/null; then
log_error "Please install kapp. e.g. 'brew tap k14s/tap && brew install kapp' for MacOS"
exit 1
fi
if ! command -v kubectl >/dev/null; then
log_error "Please install kubectl. e.g. 'brew install kubectl' for MacOS"
exit 1
fi
cd "$pinniped_path" || exit 1
if [[ ! -f Dockerfile || ! -d deploy ]]; then
echo "$pinniped_path does not appear to be the path to the source code repo directory"
exit 1
fi
if [[ ! -d "$pinniped_ci_path/pipelines/shared-helpers" ]]; then
echo "$pinniped_ci_path does not appear to be the path to the ci repo directory"
exit 1
fi
echo "Deleting running kind clusters to prepare a clean slate for the install+uninstall test..."
kind delete cluster --name pinniped
echo "Creating a kind cluster..."
kind create cluster --name pinniped
registry="docker.io"
repo="test/build"
registry_repo="$registry/$repo"
tag=$(uuidgen) # always a new tag to force K8s to reload the image on redeploy
if [[ "$skip_build" == "yes" ]]; then
most_recent_tag=$(docker images "$repo" --format "{{.Tag}}" | head -1)
if [[ -n "$most_recent_tag" ]]; then
tag="$most_recent_tag"
do_build=no
else
# Oops, there was no previous build. Need to build anyway.
do_build=yes
fi
else
do_build=yes
fi
registry_repo_tag="${registry_repo}:${tag}"
if [[ "$do_build" == "yes" ]]; then
# Rebuild the code
echo "Docker building the app..."
docker build . --tag "$registry_repo_tag"
fi
# Load it into the cluster
echo "Loading the app's container image into the kind cluster..."
kind load docker-image "$registry_repo_tag" --name pinniped
cat <<EOF >/tmp/uninstall-test-env
# The following env vars should be set before running $pinniped_ci_path/pipelines/shared-tasks/run-uninstall-test/run-uninstall-test.sh
export IMAGE_REPO="$registry_repo"
export IMAGE_TAG="$tag"
EOF
echo "Done!"
echo
echo "Ready to run an uninstall test."
echo " cd $pinniped_path"
echo "Then either"
echo " source /tmp/uninstall-test-env && $pinniped_ci_path/pipelines/shared-tasks/run-uninstall-test/run-uninstall-test.sh"
echo "or"
echo " source /tmp/uninstall-test-env && $pinniped_ci_path/pipelines/shared-tasks/run-uninstall-test/run-uninstall-from-existing-namespace-test.sh"
echo
echo "When you're finished, use 'kind delete cluster --name pinniped to tear down the cluster."
+248
View File
@@ -0,0 +1,248 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
# Assuming that you have somehow got your hands on a remote GKE or kind cluster,
# and that you have an admin kubeconfig file for it,
# and that you have already built/pushed the Pinniped container image that you would like to test,
# then you can use this script to deploy in preparation for integration or manual testing.
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
function log_note() {
GREEN='\033[0;32m'
NC='\033[0m'
if [[ ${COLORTERM:-unknown} =~ ^(truecolor|24bit)$ ]]; then
echo -e "${GREEN}$*${NC}"
else
echo "$*"
fi
}
function log_error() {
RED='\033[0;31m'
NC='\033[0m'
if [[ ${COLORTERM:-unknown} =~ ^(truecolor|24bit)$ ]]; then
echo -e "🙁${RED} Error: $* ${NC}"
else
echo ":( Error: $*"
fi
}
function check_dependency() {
if ! command -v "$1" >/dev/null; then
log_error "Missing dependency..."
log_error "$2"
exit 1
fi
}
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
#
# Handle argument parsing and help message
#
help=no
kubeconfig=""
image_tag=""
image_repo=""
pinniped_repo=""
cluster_type=""
image_digest=""
while (("$#")); do
case "$1" in
-h | --help)
help=yes
shift
;;
-k | --kubeconfig)
shift
# If there are no more command line arguments, or there is another command line argument but it starts with a dash, then error
if [[ "$#" == "0" || "$1" == -* ]]; then
log_error "-k|--kubeconfig requires a kubeconfig path to be specified"
exit 1
fi
kubeconfig=$1
shift
;;
-t | --image-tag)
shift
# If there are no more command line arguments, or there is another command line argument but it starts with a dash, then error
if [[ "$#" == "0" || "$1" == -* ]]; then
log_error "-t|--image-tag requires a tag to be specified"
exit 1
fi
image_tag=$1
shift
;;
-d | --image-digest)
shift
# If there are no more command line arguments, or there is another command line argument but it starts with a dash, then error
if [[ "$#" == "0" || "$1" == -* ]]; then
log_error "--d|--image-digest requires a digest to be specified"
exit 1
fi
image_digest=$1
shift
;;
-r | --image-repo)
shift
# If there are no more command line arguments, or there is another command line argument but it starts with a dash, then error
if [[ "$#" == "0" || "$1" == -* ]]; then
log_error "-r|--image-repo requires an image repo to be specified"
exit 1
fi
image_repo=$1
shift
;;
-p | --pinniped-repo)
shift
# If there are no more command line arguments, or there is another command line argument but it starts with a dash, then error
if [[ "$#" == "0" || "$1" == -* ]]; then
log_error "-p|--pinniped-repo requires a path to the pinniped repo to be specified"
exit 1
fi
pinniped_repo=$1
shift
;;
-c | --cluster-type)
shift
# If there are no more command line arguments, or there is another command line argument but it starts with a dash, then error
if [[ "$#" == "0" || "$1" == -* ]]; then
log_error "-c|--cluster-type requires the type of the cluster to be specified"
exit 1
fi
cluster_type=$1
shift
;;
-*)
log_error "Unsupported flag $1" >&2
exit 1
;;
*)
log_error "Unsupported positional arg $1" >&2
exit 1
;;
esac
done
# Note that if you are using a remote kind cluster then it might be more convenient to use this public repo:
# ghcr.io/pinniped-ci-bot/manual-test-pinniped-images
# You can give yourself permission to push to that repo at:
# https://github.com/users/pinniped-ci-bot/packages/container/manual-test-pinniped-images/settings
default_image_repo="gcr.io/$PINNIPED_GCP_PROJECT/manual-test-pinniped-images"
default_image_tag="latest"
if [[ "$help" == "yes" ]]; then
me="$(basename "${BASH_SOURCE[0]}")"
log_note "Usage:"
log_note " $me [flags]"
log_note
log_note "Flags:"
log_note " -h, --help: print this usage"
log_note " -k, --kubeconfig: path to the kubeconfig for your cluster (required)"
log_note " -c, --cluster-type: the type of cluster targeted by the kubeconfig, either 'gke' or 'kind' (required)"
log_note " -r, --image-repo: image registry/repository for Pinniped server container image to deploy (default: $default_image_repo)"
log_note " -t, --image-tag: image tag for Pinniped server container image to deploy (default: $default_image_tag)"
log_note " -d, --image-digest: image digest for Pinniped server container image to deploy. Takes precedence over --image-tag."
log_note " -p, --pinniped-repo: path to pinniped git repo (default: a sibling directory called pinniped)"
exit 1
fi
if [[ "$kubeconfig" == "" ]]; then
log_error "no kubeconfig set. -k|--kubeconfig is a required option."
exit 1
fi
if [[ "$kubeconfig" != "/"* ]]; then
# If it looks like a relative path then make an an absolute path because we are going to pushd below.
kubeconfig="$(pwd)/$kubeconfig"
fi
if [[ ! -f "$kubeconfig" ]]; then
log_error "specified kubeconfig file does not exist: $kubeconfig"
exit 1
fi
if [[ "$cluster_type" != "gke" && "$cluster_type" != "kind" && "$cluster_type" != "aks" && "$cluster_type" != "eks" ]]; then
log_error "specified cluster type must be 'kind', 'eks', 'aks', or 'gke'. -c|--cluster-type is a required option."
exit 1
fi
if [[ "$pinniped_repo" == "" ]]; then
pinniped_repo="$ROOT/../pinniped"
log_note "no pinniped repo path set, defaulting to $pinniped_repo"
fi
if [[ ! (-d "$pinniped_repo" && -d "$pinniped_repo/deploy" && -d "$pinniped_repo/test/cluster_capabilities") ]]; then
log_error "$pinniped_repo does not appear to contain the pinniped source code repo"
fi
if [[ "$image_repo" == "" ]]; then
image_repo="$default_image_repo"
log_note "no image repo set, defaulting to $image_repo"
fi
if [[ "$image_tag" == "" ]]; then
image_tag="$default_image_tag"
log_note "no image tag set, defaulting to $image_tag"
fi
cluster_capabilities_path="$pinniped_repo/test/cluster_capabilities/$cluster_type.yaml"
if [[ ! -f "$cluster_capabilities_path" ]]; then
log_error "cluster type capabilities file does not exist: $cluster_capabilities_path"
exit 1
fi
check_dependency ytt "Please install ytt. e.g. 'brew tap k14s/tap && brew install ytt' for MacOS"
check_dependency kapp "Please install kapp. e.g. 'brew tap k14s/tap && brew install kapp' for MacOS"
check_dependency kubectl "Please install kubectl. e.g. 'brew install kubectl' for MacOS"
check_dependency htpasswd "Please install htpasswd. Should be pre-installed on MacOS. Usually found in 'apache2-utils' package for linux."
check_dependency openssl "Please install openssl. Should be pre-installed on MacOS."
check_dependency nmap "Please install nmap. e.g. 'brew install nmap' for MacOS"
#
# Finished checking arguments and dependencies. Now actually do the work...
#
export KUBECONFIG="$kubeconfig"
export IMAGE_TAG="$image_tag"
export IMAGE_REPO="$image_repo"
if [[ "$image_digest" != "" ]]; then
export IMAGE_DIGEST="$image_digest"
fi
pushd "$pinniped_repo" >/dev/null
PINNIPED_TEST_CLUSTER_CAPABILITY_FILE="${cluster_capabilities_path}" \
DEPLOY_LOCAL_USER_AUTHENTICATOR=yes \
DEPLOY_TEST_TOOLS=yes \
CONCIERGE_APP_NAME="concierge" \
CONCIERGE_NAMESPACE="concierge" \
SUPERVISOR_APP_NAME="supervisor" \
SUPERVISOR_NAMESPACE="supervisor" \
USE_LOAD_BALANCERS_FOR_DEX_AND_SUPERVISOR="yes" \
"$ROOT/pipelines/shared-helpers/prepare-cluster-for-integration-tests.sh"
popd >/dev/null
log_note
log_note "🚀 Ready to run integration tests! For example..."
case "$cluster_type" in
gke | aks | eks)
log_note "KUBECONFIG='$KUBECONFIG' TEST_ENV_PATH='/tmp/integration-test-env' SOURCE_PATH='$pinniped_repo' $ROOT/pipelines/shared-tasks/run-integration-tests/task.sh"
;;
kind)
log_note "KUBECONFIG='$KUBECONFIG' TEST_ENV_PATH='/tmp/integration-test-env' SOURCE_PATH='$pinniped_repo' START_GCLOUD_PROXY=yes GCP_PROJECT=$PINNIPED_GCP_PROJECT GCP_ZONE=us-central1-b $ROOT/pipelines/shared-tasks/run-integration-tests/task.sh"
;;
*)
log_error "Huh? Should never get here."
;;
esac
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
# Copyright 2021-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
instance_name="${REMOTE_INSTANCE_NAME:-${USER}}"
instance_user="${REMOTE_INSTANCE_USERNAME:-${USER}}"
project="$PINNIPED_GCP_PROJECT"
zone="us-central1-b"
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Create a VM called $instance_name with some reasonable compute power and disk.
echo "Creating VM with name $instance_name..."
gcloud compute instances create "$instance_name" \
--project="$project" --zone="$zone" \
--machine-type="e2-standard-8" \
--boot-disk-size="40GB" --boot-disk-type="pd-ssd" --boot-disk-device-name="$instance_name"
# Give a little time for the server to be ready.
while true; do
sleep 5
if ! "$here"/ssh.sh ls; then
echo "Waiting for VM to be accessible via ssh..."
else
echo "VM ready!"
break
fi
done
# Copy the deps script to the new VM.
echo "Copying deps.sh to $instance_name..."
gcloud compute scp "$here"/lib/deps.sh "$instance_user@$instance_name":/tmp \
--project="$project" --zone="$zone"
# Run the deps script on the new VM.
"$here"/ssh.sh /tmp/deps.sh
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
# Copyright 2021-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
instance_name="${REMOTE_INSTANCE_NAME:-${USER}}"
project="$PINNIPED_GCP_PROJECT"
zone="us-central1-b"
# Delete the instance forever. Will prompt for confirmation.
echo "Destroying VM $instance_name..."
gcloud compute instances delete "$instance_name" \
--delete-disks="all" \
--project="$project" --zone="$zone"
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env bash
# Copyright 2021-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -exuo pipefail
# Start in the user's home directory.
cd
# Install brew pre-reqs documented at https://docs.brew.sh/Homebrew-on-Linux#requirements
sudo apt-get update && sudo sudo apt-get install build-essential procps curl file git -y
# Brew installer command from https://brew.sh. Note that CI=1 turns off an interactive prompt.
CI=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
# The installer prints more instructions. It advises you to add brew to profile and install gcc.
echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"' >>$HOME/.profile
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
brew install gcc
# Install go.
brew install go
# On linux go really wants gcc5 to also be installed for some reason.
brew install gcc@5
# Get the Go linter.
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.1
# Install and configure zsh and plugins.
brew install zsh zsh-history-substring-search
brew install fasd fzf
/home/linuxbrew/.linuxbrew/opt/fzf/install --all --no-bash --no-fish
# Install https://ohmyz.sh
export PATH=$PATH:/home/linuxbrew/.linuxbrew/bin
CHSH=no RUNZSH=no KEEP_ZSHRC=yes sh -c "$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"
# Install some plugins.
git clone --depth=1 https://github.com/romkatv/powerlevel10k.git "$HOME"/.oh-my-zsh/custom/themes/powerlevel10k
git clone https://github.com/zsh-users/zsh-autosuggestions "$HOME"/.oh-my-zsh/custom/plugins/zsh-autosuggestions
git clone https://github.com/TamCore/autoupdate-oh-my-zsh-plugins "$HOME"/.oh-my-zsh/plugins/autoupdate
git clone https://github.com/zdharma-continuum/fast-syntax-highlighting.git "$HOME"/.oh-my-zsh/custom/plugins/fast-syntax-highlighting
# Get decent .zshrc and .p10k.zsh files.
curl -fsSL https://gist.githubusercontent.com/cfryanr/c84ca9e3fe519b5a7f07426ecc7e3a7c/raw >"$HOME"/.zshrc
curl -fsSL https://gist.githubusercontent.com/cfryanr/3e55b770b9be485bd8671377ce04a3f1/raw >"$HOME"/.p10k.zsh
# Change the user's default shell.
sudo chsh -s /home/linuxbrew/.linuxbrew/bin/zsh "$USER"
# Get some other useful config files.
curl -fsSL https://gist.githubusercontent.com/cfryanr/153e167a1f2c20934fbc4dc32bbec8f2/raw >"$HOME"/.gitconfig
curl -fsSL https://gist.githubusercontent.com/cfryanr/80ada8af9a78f08b368327401ea80b6c/raw >"$HOME"/.git-authors
# Install other useful packages.
brew tap homebrew/command-not-found
brew tap vmware-tanzu/carvel
brew install ytt kbld kapp imgpkg kwt vendir
brew install git git-duet/tap/git-duet pre-commit gh
brew install k9s kind kubectl kubectx stern
brew install exa acarl005/homebrew-formulas/ls-go ripgrep procs bat tokei git-delta dust fd httpie chroma
brew install watch htop wget
brew install jesseduffield/lazydocker/lazydocker ctop dive
brew install jq yq
brew install grip
brew install aws-iam-authenticator
brew install step cfssl
brew install nmap
sudo apt-get install apache2-utils rsync -y
# Install Chrome
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
sudo apt install ./google-chrome-stable_current_amd64.deb -y
rm ./google-chrome-stable_current_amd64.deb
google-chrome --version
mkdir "$HOME"/bin
# Install docker according to procedure from https://docs.docker.com/engine/install/debian/
sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release -y
curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
sudo apt-get update
sudo apt-get install docker-ce docker-ce-cli containerd.io -y
sudo usermod -aG docker "$USER"
sudo systemctl enable docker.service
sudo systemctl enable containerd.service
# Set up the Pinniped repo
mkdir workspace
pushd workspace
ssh-keyscan -H github.com >> $HOME/.ssh/known_hosts
# This assumes that you used `--ssh-flag=-A` when using `gcloud compute ssh` to log in to the host,
# which will forward your ssh identities.
git clone git@github.com:vmware-tanzu/pinniped.git
pushd pinniped
pre-commit install
popd
popd
set +x
echo
echo "Successfully installed deps!"
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
# Copyright 2022-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
# This is similar to rsync.sh, but with the src and dest flipped at the end.
# It will copy all changes from the remote workstation back to your local machine (overwriting your local changes).
set -euo pipefail
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
SRC_DIR=${SRC_DIR:-"$HOME/workspace/pinniped"}
src_dir_parent=$(dirname "$SRC_DIR")
dest_dir="./workspace/pinniped"
instance_name="${REMOTE_INSTANCE_NAME:-${USER}}"
instance_user="${REMOTE_INSTANCE_USERNAME:-${USER}}"
project="$PINNIPED_GCP_PROJECT"
zone="us-central1-b"
config_file="/tmp/gcp-ssh-config"
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if [[ ! -d "$SRC_DIR" ]]; then
echo "ERROR: $SRC_DIR does not exist"
exit 1
fi
# Get the ssh fingerprints of all the GCP VMs.
gcloud compute config-ssh --ssh-config-file="$config_file" \
--project="$project" >/dev/null
cd "$SRC_DIR"
local_commit=$(git rev-parse --short HEAD)
remote_commit=$("$here"/ssh.sh "cd $dest_dir; git rev-parse --short HEAD" 2>/dev/null | tr -dc '[:print:]')
if [[ -z "$local_commit" || -z "$remote_commit" ]]; then
echo "ERROR: Could not determine currently checked out git commit sha"
exit 1
fi
if [[ "$local_commit" != "$remote_commit" ]]; then
echo "ERROR: Local and remote repos are not on the same commit. This is usually a mistake."
echo "Local was $SRC_DIR at *${local_commit}*"
echo "Remote was ${instance_name}:${dest_dir} at *${remote_commit}*"
exit 1
fi
# Skip large files because they are probably compiled binaries.
# Also skip other common filenames that we wouldn't need to sync.
echo "Starting rsync from remote to local for $SRC_DIR..."
rsync \
--progress --delete --archive --compress --human-readable \
--max-size 200K \
--exclude .git/ --exclude .idea/ --exclude .DS_Store --exclude '*.test' --exclude '*.out' \
--rsh "ssh -F $config_file" \
"${instance_user}@${instance_name}.${zone}.${project}:$dest_dir" "$src_dir_parent"
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
# Copyright 2021 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
SRC_DIR=${SRC_DIR:-"$HOME/workspace/pinniped"}
dest_dir="./workspace"
instance_name="${REMOTE_INSTANCE_NAME:-${USER}}"
instance_user="${REMOTE_INSTANCE_USERNAME:-${USER}}"
project="$PINNIPED_GCP_PROJECT"
zone="us-central1-b"
config_file="/tmp/gcp-ssh-config"
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if [[ ! -d "$SRC_DIR" ]]; then
echo "ERROR: $SRC_DIR does not exist"
exit 1
fi
# Get the ssh fingerprints of all the GCP VMs.
gcloud compute config-ssh --ssh-config-file="$config_file" \
--project="$project" >/dev/null
cd "$SRC_DIR"
local_commit=$(git rev-parse --short HEAD)
remote_commit=$("$here"/ssh.sh "cd $dest_dir/pinniped; git rev-parse --short HEAD" 2>/dev/null | tr -dc '[:print:]')
if [[ -z "$local_commit" || -z "$remote_commit" ]]; then
echo "ERROR: Could not determine currently checked out git commit sha"
exit 1
fi
if [[ "$local_commit" != "$remote_commit" ]]; then
echo "ERROR: Local and remote repos are not on the same commit. This is usually a mistake."
echo "Local was $SRC_DIR at *${local_commit}*"
echo "Remote was ${instance_name}:${dest_dir}/pinniped at *${remote_commit}*"
exit 1
fi
# Skip large files because they are probably compiled binaries.
# Also skip other common filenames that we wouldn't need to sync.
echo "Starting rsync for $SRC_DIR..."
rsync \
--progress --delete --archive --compress --human-readable \
--max-size 200K \
--exclude .git/ --exclude .idea/ --exclude .DS_Store --exclude '*.test' --exclude '*.out' \
--rsh "ssh -F $config_file" \
"$SRC_DIR" "${instance_user}@${instance_name}.${zone}.${project}:$dest_dir"
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
# Copyright 2021-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
instance_name="${REMOTE_INSTANCE_NAME:-${USER}}"
instance_user="${REMOTE_INSTANCE_USERNAME:-${USER}}"
project="$PINNIPED_GCP_PROJECT"
zone="us-central1-b"
# Run ssh with identities forwarded so you can use them with git on the remote host.
# Optionally run an arbitrary command on the remote host.
# By default, start an interactive session.
gcloud compute ssh --ssh-flag=-A "$instance_user@$instance_name" \
--project="$project" --zone="$zone" -- "$@"
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Copyright 2021-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
instance_name="${REMOTE_INSTANCE_NAME:-${USER}}"
project="$PINNIPED_GCP_PROJECT"
zone="us-central1-b"
# Start an instance which was previously stopped to save money.
echo "Starting VM $instance_name..."
gcloud compute instances start "$instance_name" \
--project="$project" --zone="$zone"
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Copyright 2021-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
if [[ -z "${PINNIPED_GCP_PROJECT:-}" ]]; then
echo "PINNIPED_GCP_PROJECT env var must be set"
exit 1
fi
instance_name="${REMOTE_INSTANCE_NAME:-${USER}}"
project="$PINNIPED_GCP_PROJECT"
zone="us-central1-b"
# Stop the instance, to save money, in a way that it can be restarted.
echo "Stopping VM $instance_name..."
gcloud compute instances stop "$instance_name" \
--project="$project" --zone="$zone"
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
# This script will prepare to run the integration tests and then run them.
# Is is a wrapper for prepare-for-integration-tests.sh to make it convenient
# to run the integration tests, potentially running them repeatedly.
set -euo pipefail
help=no
skip_build=no
delete_kind_cluster=no
PARAMS=""
while (("$#")); do
case "$1" in
-h | --help)
help=yes
shift
;;
-s | --skip-build)
skip_build=yes
shift
;;
-c | --from-clean-cluster)
delete_kind_cluster=yes
shift
;;
-*)
echo "Error: Unsupported flag $1" >&2
exit 1
;;
*)
PARAMS="$PARAMS $1"
shift
;;
esac
done
eval set -- "$PARAMS"
if [[ "$help" == "yes" ]]; then
me="$(basename "${BASH_SOURCE[0]}")"
echo "Usage:"
echo " $me [flags] [path/to/pinniped]"
echo
echo " path/to/pinniped default: \$PWD ($PWD)"
echo
echo "Flags:"
echo " -h, --help: print this usage"
echo " -s, --skip-build: reuse the most recently built image of the app instead of building"
echo " -c, --from-clean-cluster: delete and rebuild the kind cluster before running tests"
exit 1
fi
pinniped_path="${1-$PWD}"
cd "$pinniped_path" || exit 1
if [[ ! -f Dockerfile || ! -d deploy ]]; then
echo "$pinniped_path does not appear to be the path to the source code repo directory"
exit 1
fi
if ! command -v kind >/dev/null; then
echo "Please install kind. e.g. 'brew install kind' for MacOS"
exit 1
fi
if [[ "$delete_kind_cluster" == "yes" ]]; then
echo "Deleting running kind clusters to prepare a clean slate..."
"$pinniped_path"/hack/kind-down.sh
fi
if [[ "$skip_build" == "yes" ]]; then
"$pinniped_path"/hack/prepare-for-integration-tests.sh --skip-build
else
"$pinniped_path"/hack/prepare-for-integration-tests.sh
fi
source /tmp/integration-test-env
ulimit -n 512
echo
echo "Running integration tests..."
go test -race -v -count 1 -timeout 0 ./test/integration
echo "ALL INTEGRATION TESTS PASSED"
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env bash
# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Define some env vars
source "$script_dir/fly-helpers.sh"
# Install the fly cli if needed
if [[ ! -f "$FLY_CLI" ]]; then
curl -fL "$CONCOURSE_URL/api/v1/cli?arch=amd64&platform=darwin" -o "$FLY_CLI"
chmod 755 "$FLY_CLI"
fi
if ! $FLY_CLI targets | tr -s ' ' | cut -f1 -d ' ' | grep -q "$CONCOURSE_TARGET"; then
# Create the target if needed
$FLY_CLI --target "$CONCOURSE_TARGET" login \
--team-name "$CONCOURSE_TEAM" --concourse-url "$CONCOURSE_URL"
else
# Login if needed
if ! $FLY_CLI --target "$CONCOURSE_TARGET" status; then
$FLY_CLI --target "$CONCOURSE_TARGET" login
fi
fi
# Upgrade fly if needed
$FLY_CLI --target "$CONCOURSE_TARGET" sync