From bc73505e356cf603c387a8ee88bb98d8a0662ee0 Mon Sep 17 00:00:00 2001 From: Joshua Casey Date: Wed, 20 Nov 2024 09:55:24 -0600 Subject: [PATCH] Easily enable kind audit logs with ENABLE_AUDIT_LOGGING=true ./hack/kind-up.sh --- hack/kind-up.sh | 6 +- .../kind-config/metadata-audit-policy.yaml | 4 + hack/lib/kind-config/single-node.yaml | 96 ++++++++++++------- 3 files changed, 72 insertions(+), 34 deletions(-) create mode 100644 hack/lib/kind-config/metadata-audit-policy.yaml diff --git a/hack/kind-up.sh b/hack/kind-up.sh index 7fdb90d03..b0fce803c 100755 --- a/hack/kind-up.sh +++ b/hack/kind-up.sh @@ -37,8 +37,12 @@ if [[ "${PINNIPED_USE_LOCAL_KIND_REGISTRY:-}" != "" ]]; then use_kind_registry="--file=${ROOT}/hack/lib/kind-config/kind-registry-overlay.yaml" fi +cp "${ROOT}/hack/lib/kind-config/metadata-audit-policy.yaml" /tmp/metadata-audit-policy.yaml + # Do not quote ${use_kind_registry} ${use_contour_registry} in this command because they might be empty. -ytt ${use_kind_registry} ${use_contour_registry} --file="${ROOT}"/hack/lib/kind-config/single-node.yaml >/tmp/kind-config.yaml +ytt ${use_kind_registry} ${use_contour_registry} \ + --data-value-yaml enable_audit_logs=${ENABLE_KIND_AUDIT_LOGS:-false} \ + --file="${ROOT}"/hack/lib/kind-config/single-node.yaml >/tmp/kind-config.yaml # To choose a specific version of kube, add this option to the command below: `--image kindest/node:v1.28.0`. # To use the "latest-main" version of kubernetes builds by the pipeline, use `--image ghcr.io/pinniped-ci-bot/kind-node-image:latest` diff --git a/hack/lib/kind-config/metadata-audit-policy.yaml b/hack/lib/kind-config/metadata-audit-policy.yaml new file mode 100644 index 000000000..67ec4611e --- /dev/null +++ b/hack/lib/kind-config/metadata-audit-policy.yaml @@ -0,0 +1,4 @@ +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Metadata diff --git a/hack/lib/kind-config/single-node.yaml b/hack/lib/kind-config/single-node.yaml index f71785087..71202f09c 100644 --- a/hack/lib/kind-config/single-node.yaml +++ b/hack/lib/kind-config/single-node.yaml @@ -1,46 +1,76 @@ +#@ load("@ytt:data", "data") + kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraPortMappings: - protocol: TCP - # This same port number is hardcoded in the integration test setup - # when creating a Service on a kind cluster. It is used to talk to - # the supervisor app via HTTPS. + #! This same port number is hardcoded in the integration test setup + #! when creating a Service on a kind cluster. It is used to talk to + #! the supervisor app via HTTPS. containerPort: 31243 hostPort: 12344 listenAddress: 127.0.0.1 - protocol: TCP - # This same port number is hardcoded in the integration test setup - # when creating a Service on a kind cluster. It is used to talk to - # the Dex app. + #! This same port number is hardcoded in the integration test setup + #! when creating a Service on a kind cluster. It is used to talk to + #! the Dex app. containerPort: 31235 hostPort: 12346 listenAddress: 127.0.0.1 -# Kind v0.12.0 ignores kubeadm.k8s.io/v1beta2 for Kube v1.23+ but uses it for older versions of Kube. -# Previous versions of Kind would use kubeadm.k8s.io/v1beta2 for all versions of Kube including 1.23. -# To try to maximize compatibility with various versions of Kind and Kube, define this -# ClusterConfiguration twice and hope that Kind will use the one that it likes for the given version -# of Kube, and ignore the one that it doesn't like. This seems to work, at least for Kind v0.12.0. -kubeadmConfigPatches: -- | - apiVersion: kubeadm.k8s.io/v1beta2 - kind: ClusterConfiguration - apiServer: - extraArgs: - # To make sure the endpoints on our service are correct (this mostly matters for kubectl based - # installs where kapp is not doing magic changes to the deployment and service selectors). - # Setting this field to true makes it so that the API service will do the service cluster IP - # to endpoint IP translations internally instead of relying on the network stack (i.e. kube-proxy). - # The logic inside the API server is very straightforward - randomly pick an IP from the list - # of available endpoints. This means that over time, all endpoints associated with the service - # are exercised. For whatever reason, leaving this as false (i.e. use kube-proxy) appears to - # hide some network misconfigurations when used internally by the API server aggregation layer. - enable-aggregator-routing: "true" -- | - apiVersion: kubeadm.k8s.io/v1beta3 - kind: ClusterConfiguration - apiServer: - extraArgs: - # See comment above. - enable-aggregator-routing: "true" + + + #! Kind v0.12.0 ignores kubeadm.k8s.io/v1beta2 for Kube v1.23+ but uses it for older versions of Kube. + #! Previous versions of Kind would use kubeadm.k8s.io/v1beta2 for all versions of Kube including 1.23. + #! To try to maximize compatibility with various versions of Kind and Kube, define this + #! ClusterConfiguration twice and hope that Kind will use the one that it likes for the given version + #! of Kube, and ignore the one that it doesn't like. This seems to work, at least for Kind v0.12.0. + kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: ClusterConfiguration + apiServer: + extraArgs: + #! To make sure the endpoints on our service are correct (this mostly matters for kubectl based + #! installs where kapp is not doing magic changes to the deployment and service selectors). + #! Setting this field to true makes it so that the API service will do the service cluster IP + #! to endpoint IP translations internally instead of relying on the network stack (i.e. kube-proxy). + #! The logic inside the API server is very straightforward - randomly pick an IP from the list + #! of available endpoints. This means that over time, all endpoints associated with the service + #! are exercised. For whatever reason, leaving this as false (i.e. use kube-proxy) appears to + #! hide some network misconfigurations when used internally by the API server aggregation layer. + enable-aggregator-routing: "true" + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + apiServer: + extraArgs: + # See comment above. + enable-aggregator-routing: "true" + #@ if data.values.enable_audit_logs: + - | + kind: ClusterConfiguration + apiServer: + #! enable auditing flags on the API server + extraArgs: + audit-log-path: /var/log/kubernetes/kube-apiserver-audit.log + audit-policy-file: /etc/kubernetes/policies/audit-policy.yaml + #! mount new files / directories on the control plane + extraVolumes: + - name: audit-policies + hostPath: /etc/kubernetes/policies + mountPath: /etc/kubernetes/policies + readOnly: true + pathType: "DirectoryOrCreate" + - name: "audit-logs" + hostPath: "/var/log/kubernetes" + mountPath: "/var/log/kubernetes" + readOnly: false + pathType: DirectoryOrCreate + #! mount the local file on the control plane + extraMounts: + - hostPath: /tmp/metadata-audit-policy.yaml + containerPath: /etc/kubernetes/policies/audit-policy.yaml + readOnly: true + #@ end