CI changes for testing GOFIPS140 builds in PR pipeline

Signed-off-by: Ryan Richard <richardry@vmware.com>
This commit is contained in:
Ryan Richard
2026-07-06 10:31:26 -07:00
parent 82cb634931
commit d9d65c97af
6 changed files with 298 additions and 5 deletions
+196 -2
View File
@@ -260,6 +260,15 @@ resources:
password: ((ci-ghcr-pusher-token))
tag: latest
- name: ci-gofips140-build-image
type: registry-image
icon: docker
source:
repository: ((ci-ghcr-registry))/test-ci-gofips140-build
username: ((ci-ghcr-pusher-username))
password: ((ci-ghcr-pusher-token))
tag: latest
- name: ci-test-image
type: registry-image
icon: docker
@@ -278,6 +287,15 @@ resources:
password: ((ci-ghcr-pusher-token))
tag: latest
- name: ci-gofips140-test-image
type: registry-image
icon: docker
source:
repository: ((ci-ghcr-registry))/test-ci-gofips140-test-binaries
username: ((ci-ghcr-pusher-username))
password: ((ci-ghcr-pusher-token))
tag: latest
- name: integration-test-runner-image
type: registry-image
icon: docker
@@ -400,6 +418,8 @@ jobs:
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: build-image } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: build-fips-image } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: build-test-fips-image } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: build-gofips140-image } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: build-test-gofips140-image } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: build-test-binaries } }
#- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: scan-image } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: integration-test-oldest } }
@@ -407,6 +427,7 @@ jobs:
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: integration-test-latest-arm64 } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: integration-test-idps-firewalled } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: integration-test-latest-fips } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: integration-test-latest-gofips140 } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: integration-test-k8s-main } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: kubectl-apply-test } }
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, target_url: "$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME", context: integration-test-changed-api-group-latest } }
@@ -752,8 +773,7 @@ jobs:
image: ci-build-image
pinniped: pinniped-pr
# this job builds pinniped using a different dockerfile
# to test that it is fips compatible.
# this job builds pinniped using a different dockerfile to test that it is fips compatible.
# it uses go-boringcrypto rather than base go.
- name: build-fips-image
on_success: { <<: *pr-status-on-success, params: { <<: *pr-status-on-success-params, context: build-fips-image } }
@@ -797,6 +817,49 @@ jobs:
params:
image: image/image.tar
# This job builds pinniped using GOFIPS140, Go's native FIPS compiler.
- name: build-gofips140-image
on_success: { <<: *pr-status-on-success, params: { <<: *pr-status-on-success-params, context: build-gofips140-image } }
on_failure: { <<: *pr-status-on-failure, params: { <<: *pr-status-on-failure-params, context: build-gofips140-image } }
on_error: { <<: *pr-status-on-error, params: { <<: *pr-status-on-error-params, context: build-gofips140-image } }
on_abort: { <<: *pr-status-on-abort, params: { <<: *pr-status-on-abort-params, context: build-gofips140-image } }
public: true # all logs are publicly visible
plan:
- in_parallel:
- get: pinniped-pr
trigger: true
version: every
passed: [ start ]
- get: pinniped-ci
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, context: build-gofips140-image } }
- task: build-gofips140-image
privileged: true
config:
platform: linux
image_resource:
type: registry-image
source:
repository: concourse/oci-build-task
inputs:
- name: pinniped-ci
- name: pinniped-pr
outputs:
- name: image
run:
path: build
caches:
- path: cache
params:
CONTEXT: pinniped-pr
UNPACK_ROOTFS: true
BUILD_ARG_GOFIPS140: certified
- task: confirm-built-with-gofips140
file: pinniped-ci/pipelines/shared-tasks/confirm-built-with-gofips140/task.yml
- put: ci-gofips140-build-image
get_params: { skip_download: true }
params:
image: image/image.tar
- name: build-test-image
on_success: { <<: *pr-status-on-success, params: { <<: *pr-status-on-success-params, context: build-test-binaries } }
on_failure: { <<: *pr-status-on-failure, params: { <<: *pr-status-on-failure-params, context: build-test-binaries } }
@@ -877,6 +940,47 @@ jobs:
params:
image: image/image.tar
- name: build-test-gofips140-image
on_success: { <<: *pr-status-on-success, params: { <<: *pr-status-on-success-params, context: build-test-gofips140-image } }
on_failure: { <<: *pr-status-on-failure, params: { <<: *pr-status-on-failure-params, context: build-test-gofips140-image } }
on_error: { <<: *pr-status-on-error, params: { <<: *pr-status-on-error-params, context: build-test-gofips140-image } }
on_abort: { <<: *pr-status-on-abort, params: { <<: *pr-status-on-abort-params, context: build-test-gofips140-image } }
public: true # all logs are publicly visible
plan:
- in_parallel:
- get: pinniped-pr
trigger: true
version: every
passed: [ start ]
- get: pinniped-ci
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, context: build-test-gofips140-image } }
- in_parallel:
- task: build-test-fips-image
privileged: true
config:
platform: linux
image_resource:
type: registry-image
source:
repository: concourse/oci-build-task
inputs:
- name: pinniped-pr
- name: pinniped-ci
outputs:
- name: image
run:
path: build
caches:
- path: cache
params:
CONTEXT: pinniped-pr
BUILD_ARG_GOFIPS140: certified
DOCKERFILE: pinniped-ci/pipelines/shared-helpers/test-binaries-image/Dockerfile
- put: ci-gofips140-test-image
get_params: { skip_download: true }
params:
image: image/image.tar
- name: ready-for-int # fan-in to make pass constraints for the rest of the pipeline easier to reason about
public: true # all logs are publicly visible
plan:
@@ -894,6 +998,8 @@ jobs:
- build-test-image
- build-fips-image
- build-test-fips-image
- build-gofips140-image
- build-test-gofips140-image
trigger: true
- get: ci-build-image
passed: [ check-image-version ]
@@ -907,10 +1013,18 @@ jobs:
passed: [ build-fips-image ]
params:
skip_download: true
- get: ci-gofips140-build-image
passed: [ build-gofips140-image ]
params:
skip_download: true
- get: ci-fips-test-image
passed: [ build-test-fips-image ]
params:
skip_download: true
- get: ci-gofips140-test-image
passed: [ build-test-gofips140-image ]
params:
skip_download: true
# Splitting the integration tests into two columns to reduce parallelism of those jobs,
# so we can have a single worker to reduce costs, and to reduce the number of jobs
@@ -937,7 +1051,9 @@ jobs:
- check-image-version
- build-test-image
- build-fips-image
- build-gofips140-image
- build-test-fips-image
- build-test-gofips140-image
trigger: true
- get: ci-build-image
passed: [ check-image-version ]
@@ -951,10 +1067,18 @@ jobs:
passed: [ build-fips-image ]
params:
skip_download: true
- get: ci-gofips140-build-image
passed: [ build-gofips140-image ]
params:
skip_download: true
- get: ci-fips-test-image
passed: [ build-test-fips-image ]
params:
skip_download: true
- get: ci-gofips140-test-image
passed: [ build-test-gofips140-image ]
params:
skip_download: true
# - name: scan-image
# on_success: { <<: *pr-status-on-success, params: { <<: *pr-status-on-success-params, context: scan-image } }
@@ -1403,6 +1527,75 @@ jobs:
params:
<<: *gcp_account_params
- name: integration-test-latest-gofips140
on_success: { <<: *pr-status-on-success, params: { <<: *pr-status-on-success-params, context: integration-test-latest-gofips140 } }
on_failure: { <<: *pr-status-on-failure, params: { <<: *pr-status-on-failure-params, context: integration-test-latest-gofips140 } }
on_error: { <<: *pr-status-on-error, params: { <<: *pr-status-on-error-params, context: integration-test-latest-gofips140 } }
on_abort: { <<: *pr-status-on-abort, params: { <<: *pr-status-on-abort-params, context: integration-test-latest-gofips140 } }
public: true # all logs are publicly visible
plan:
- in_parallel:
- get: pinniped-pr
version: every
passed: [ ready-for-int2 ]
trigger: true
- get: pinniped-ci
- get: kind-release
- get: gcloud-image
- get: k8s-app-deployer-image
- get: ci-gofips140-build-image
passed: [ ready-for-int2 ]
params:
skip_download: true
- get: ci-gofips140-test-image
passed: [ ready-for-int2 ]
- get: integration-test-runner-image
- { <<: *pr-status-on-pending, params: { <<: *pr-status-on-pending-params, context: integration-test-latest-gofips140 } }
- task: deploy-kind-instance
attempts: 5
timeout: 30m
file: pinniped-ci/pipelines/shared-tasks/deploy-kind-cluster-vm/task.yml
image: gcloud-image
params:
<<: *latest_kind_kube_version
<<: *gcp_account_params
- task: deploy
input_mapping:
ci-build-image: ci-gofips140-build-image
cluster-pool: deploy-kind-cluster-vm-output
pinniped: pinniped-pr
params:
SUPERVISOR_AND_CONCIERGE_GODEBUG_FIPS140_ONLY: "yes"
CLUSTER_CAPABILITIES_PATH: pinniped/test/cluster_capabilities/kind.yaml
file: pinniped-ci/pipelines/shared-tasks/deploy-to-integration/task.yml
image: integration-test-runner-image
- task: run-integration-tests
timeout: 75m
file: pinniped-ci/pipelines/shared-tasks/run-integration-tests/task.yml
input_mapping:
pinniped: pinniped-pr
ci-test-image: ci-gofips140-test-image
image: integration-test-runner-image
params:
START_GCLOUD_PROXY: "yes"
<<: *gcp_account_params
ensure:
task: export-cluster-diagnostics
<<: *cluster_diagnostics_task
input_mapping:
ci-build-image: ci-gofips140-build-image
cluster-pool: deploy-kind-cluster-vm-output
ensure:
task: remove-kind-instance
attempts: 2
timeout: 20m
file: pinniped-ci/pipelines/shared-tasks/remove-kind-cluster-vm/task.yml
image: gcloud-image
input_mapping:
kind-cluster-pool: deploy-kind-cluster-vm-output
params:
<<: *gcp_account_params
- name: integration-test-k8s-main
on_success: { <<: *pr-status-on-success, params: { <<: *pr-status-on-success-params, context: integration-test-k8s-main } }
on_failure: { <<: *pr-status-on-failure, params: { <<: *pr-status-on-failure-params, context: integration-test-k8s-main } }
@@ -1915,6 +2108,7 @@ jobs:
- integration-test-idps-firewalled
- integration-test-multiple-pinnipeds
- integration-test-latest-fips
- integration-test-latest-gofips140
- uninstall-test
- uninstall-existing-ns-test
# integration-test-k8s-main is purposefully excluded. Failures are informational, not blocking.
@@ -278,6 +278,22 @@ else
echo "$api_resource_output"
fi
# Save this file for possible later use. Sometimes we want to add the GODEBUG fips140=only env var.
cat <<EOF >>/tmp/add-fips140-only-overlay.yaml
#@ load("@ytt:overlay", "overlay")
#@overlay/match by=overlay.subset({"kind": "Deployment"}), expects=1
---
spec:
template:
spec:
containers:
- #@overlay/match by=overlay.all, expects=1
env:
#@overlay/append
- name: GODEBUG
value: fips140=only
EOF
# Save this file for possible later use. Sometimes we want to remove the CPU requests,
# which also means that we need to remove the limits or else Kubernetes will use the limit as
# an implicit request amount.
@@ -825,6 +841,9 @@ fi
if [[ -n "${SUPERVISOR_AND_CONCIERGE_NO_CPU_REQUEST:-}" ]]; then
concierge_optional_ytt_values+=("--file=/tmp/remove-cpu-request-overlay.yaml")
fi
if [[ -n "${SUPERVISOR_AND_CONCIERGE_GODEBUG_FIPS140_ONLY:-}" ]]; then
concierge_optional_ytt_values+=("--file=/tmp/add-fips140-only-overlay.yaml")
fi
echo "Deploying the Concierge app to the cluster..."
echo "Using ytt optional flags:" "${concierge_optional_ytt_values[@]}"
@@ -905,6 +924,9 @@ fi
if [[ -n "${SUPERVISOR_AND_CONCIERGE_NO_CPU_REQUEST:-}" ]]; then
supervisor_optional_ytt_values+=("--file=/tmp/remove-cpu-request-overlay.yaml")
fi
if [[ -n "${SUPERVISOR_AND_CONCIERGE_GODEBUG_FIPS140_ONLY:-}" ]]; then
supervisor_optional_ytt_values+=("--file=/tmp/add-fips140-only-overlay.yaml")
fi
if [[ "${SUPERVISOR_INGRESS:-no}" == "yes" && "$cluster_has_gke_backend_config" == "yes" ]]; then
supervisor_optional_ytt_values+=("--file=/tmp/add-annotations-for-gke-ingress-overlay.yaml")
fi
@@ -7,6 +7,12 @@ FROM golang:1.26.4-bookworm as build-env
WORKDIR /work
COPY . .
ARG GOPROXY
# Set this if you would like to enable Go's built-in FIPS compliance module.
# E.g. GOFIPS140=certified or GOFIPS140=v1.0.0
# See https://go.dev/doc/security/fips140 for details.
ARG GOFIPS140=off
RUN \
--mount=type=cache,target=/cache/gocache \
--mount=type=cache,target=/cache/gomodcache \
@@ -15,8 +21,8 @@ RUN \
export GOMODCACHE=/cache/gomodcache && \
export GOOS=linux && \
export GOARCH=amd64 && \
CGO_ENABLED=0 go build -v -ldflags "$(hack/get-ldflags.sh)" -o out/pinniped ./cmd/pinniped/main.go && \
CGO_ENABLED=1 go test -c -vet=off -race ./test/integration -o out/pinniped-integration-test # need cgo for race detector
CGO_ENABLED=0 GOFIPS140=$GOFIPS140 go build -v -ldflags "$(hack/get-ldflags.sh)" -o out/pinniped ./cmd/pinniped/main.go && \
CGO_ENABLED=1 GOFIPS140=$GOFIPS140 go test -c -vet=off -race ./test/integration -o out/pinniped-integration-test # need cgo for race detector
FROM scratch
COPY --from=build-env /work/out/ /usr/local/bin/
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Copyright 2026 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
# Checking whether a binary was built with GOFIPS140.
#
# Example:
#
# go version -m ./pinniped | grep 'GOFIPS140=v'
# build GOFIPS140=v1.0.0-c2097c7c
#
# go version -m ./pinniped | grep 'DefaultGODEBUG=fips140=on'
# build DefaultGODEBUG=fips140=on
# Check whether the pinniped-server binary was built with GOFIPS140.
# Look for where GOFIPS140 starts with "v" because it can also equal "off", which we don't want.
if ! go version -m './image/rootfs/usr/local/bin/pinniped-server' | grep -q 'GOFIPS140=v'; then
echo "Pinniped server binary wasn't built with GOFIPS140 enabled (no GOFIPS140=v)."
exit 1
fi
if ! go version -m './image/rootfs/usr/local/bin/pinniped-server' | grep -q 'DefaultGODEBUG=fips140=on'; then
echo "Pinniped server binary wasn't built with GOFIPS140 enabled (no DefaultGODEBUG=fips140=on)."
exit 1
fi
# Check the same for the kube-cert-agent binary.
if ! go version -m './image/rootfs/usr/local/bin/pinniped-concierge-kube-cert-agent' | grep -q 'GOFIPS140=v'; then
echo "pinniped-concierge-kube-cert-agent binary wasn't built with GOFIPS140 enabled (no GOFIPS140=v)."
exit 1
fi
if ! go version -m './image/rootfs/usr/local/bin/pinniped-concierge-kube-cert-agent' | grep -q 'DefaultGODEBUG=fips140=on'; then
echo "pinniped-concierge-kube-cert-agent binary wasn't built with GOFIPS140 enabled (no DefaultGODEBUG=fips140=on)."
exit 1
fi
# Check the ldd output to see whether we compiled a static executable or not.
pinniped_server_ldd="$(ldd './image/rootfs/usr/local/bin/pinniped-server' 2>&1)"
# If it doesn't contain this line, that means the executable was dynamic, which we don't want.
if [[ "$pinniped_server_ldd" != *"not a dynamic executable"* ]]; then
echo "pinniped server binary is a dynamic executable."
exit 1
fi
# Check the ldd output to see whether we compiled a static executable or not.
kube_cert_agent_ldd="$(ldd './image/rootfs/usr/local/bin/pinniped-concierge-kube-cert-agent' 2>&1)"
# If it doesn't contain this line, that means the executable was dynamic, which we don't want.
if [[ "$kube_cert_agent_ldd" != *"not a dynamic executable"* ]]; then
echo "kube cert agent binary is a dynamic executable."
exit 1
fi
@@ -0,0 +1,15 @@
# Copyright 2026 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
---
platform: linux
inputs:
- name: image
- name: pinniped-ci
image_resource:
type: registry-image
source:
repository: golang
tag: '1.26.4'
run:
path: pinniped-ci/pipelines/shared-tasks/confirm-built-with-gofips140/task.sh
@@ -1,4 +1,4 @@
# Copyright 2020-2025 the Pinniped contributors. All Rights Reserved.
# Copyright 2020-2026 the Pinniped contributors. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
---
@@ -33,6 +33,9 @@ params:
# only needed for unusual case of wanting to test the HTTPS_PROXY settings
FIREWALL_IDPS:
# only used when testing GOFIPS140 builds
SUPERVISOR_AND_CONCIERGE_GODEBUG_FIPS140_ONLY:
# needed when we are testing against active directory.
TEST_ACTIVE_DIRECTORY:
AWS_AD_HOST: