mirrors/pinniped
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/pinniped.git synced 2026-01-09 15:44:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Switch to a slimmer distroless base image.

Browse Source 
At a high level, it switches us to a distroless base container image, but that also includes several related bits:

- Add a writable /tmp but make the rest of our filesystems read-only at runtime.

- Condense our main server binaries into a single pinniped-server binary. This saves a bunch of space in
  the image due to duplicated library code. The correct behavior is dispatched based on `os.Args[0]`, and
  the `pinniped-server` binary is symlinked to `pinniped-concierge` and `pinniped-supervisor`.

- Strip debug symbols from our binaries. These aren't really useful in a distroless image anyway and all the
  normal stuff you'd expect to work, such as stack traces, still does.

- Add a separate `pinniped-concierge-kube-cert-agent` binary with "sleep" and "print" functionality instead of
  using builtin /bin/sleep and /bin/cat for the kube-cert-agent. This is split from the main server binary
  because the loading/init time of the main server binary was too large for the tiny resource footprint we
  established in our kube-cert-agent PodSpec. Using a separate binary eliminates this issue and the extra
  binary adds only around 1.5MiB of image size.

- Switch the kube-cert-agent code to use a JSON `{"tls.crt": "<b64 cert>", "tls.key": "<b64 key>"}` format.
  This is more robust to unexpected input formatting than the old code, which simply concatenated the files
  with some extra newlines and split on whitespace.

- Update integration tests that made now-invalid assumptions about the `pinniped-server` image.

Signed-off-by: Matt Moyer <moyerm@vmware.com>
This commit is contained in:
Matt Moyer
2021-07-26 11:18:43 -05:00
committed by Monis Khan
parent a464c81711
commit 58bbffded4
19 changed files with 538 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
internal/controller/kubecertagent/kubecertagent.go
View File

@@ -8,6 +8,7 @@ package kubecertagent
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"strings"
"time"
@@ -309,22 +310,30 @@ func (c *agentController) loadSigningKey(agentPod *corev1.Pod) error {
}
// Exec into the agent pod and cat out the certificate and the key.
combinedPEM, err := c.executor.Exec(
agentPod.Namespace, agentPod.Name,
"sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}",
)
outputJSON, err := c.executor.Exec(agentPod.Namespace, agentPod.Name, "pinniped-concierge-kube-cert-agent", "print")
if err != nil {
return fmt.Errorf("could not exec into agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
}
// Split up the output by looking for the block of newlines.
var certPEM, keyPEM string
if parts := strings.Split(combinedPEM, "\n\n\n"); len(parts) == 2 {
certPEM, keyPEM = parts[0], parts[1]
// Parse and decode the JSON output from the "pinniped-concierge-kube-cert-agent print" command.
var output struct {
Cert string `json:"tls.crt"`
Key string `json:"tls.key"`
}
if err := json.Unmarshal([]byte(outputJSON), &output); err != nil {
return fmt.Errorf("failed to decode signing cert/key JSON from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
}
certPEM, err := base64.StdEncoding.DecodeString(output.Cert)
if err != nil {
return fmt.Errorf("failed to decode signing cert base64 from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
}
keyPEM, err := base64.StdEncoding.DecodeString(output.Key)
if err != nil {
return fmt.Errorf("failed to decode signing key base64 from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
}
// Load the certificate and key into the dynamic signer.
if err := c.dynamicCertProvider.SetCertKeyContent([]byte(certPEM), []byte(keyPEM)); err != nil {
if err := c.dynamicCertProvider.SetCertKeyContent(certPEM, keyPEM); err != nil {
return fmt.Errorf("failed to set signing cert/key content from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
}
@@ -461,7 +470,7 @@ func (c *agentController) newAgentDeployment(controllerManagerPod *corev1.Pod) *
Name: "sleeper",
Image: c.cfg.ContainerImage,
ImagePullPolicy: corev1.PullIfNotPresent,
Command: []string{"/bin/sleep", "infinity"},
Command: []string{"pinniped-concierge-kube-cert-agent", "sleep"},
VolumeMounts: volumeMounts,
Env: []corev1.EnvVar{
{Name: "CERT_PATH", Value: getContainerArgByName(controllerManagerPod, "cluster-signing-cert-file", "/etc/kubernetes/ca/ca.pem")},

98
internal/controller/kubecertagent/kubecertagent_test.go
View File

@@ -104,7 +104,7 @@ func TestAgentController(t *testing.T) {
Containers: []corev1.Container{{
Name: "sleeper",
Image: "pinniped-server-image",
Command: []string{"/bin/sleep", "infinity"},
Command: []string{"pinniped-concierge-kube-cert-agent", "sleep"},
Env: []corev1.EnvVar{
{Name: "CERT_PATH", Value: "/path/to/signing.crt"},
{Name: "KEY_PATH", Value: "/path/to/signing.key"},
@@ -200,8 +200,8 @@ func TestAgentController(t *testing.T) {
}
mockExecSucceeds := func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}").
Return("test-cert\n\n\ntest-key", nil)
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
Return(`{"tls.crt": "dGVzdC1jZXJ0", "tls.key": "dGVzdC1rZXk="}`, nil) // "test-cert" / "test-key"
dynamicCert.SetCertKeyContent([]byte("test-cert"), []byte("test-key")).
Return(nil)
}
@@ -573,7 +573,7 @@ func TestAgentController(t *testing.T) {
validClusterInfoConfigMap,
},
mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}").
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
Return("", fmt.Errorf("some exec error")).
AnyTimes()
},
@@ -589,6 +589,90 @@ func TestAgentController(t *testing.T) {
LastUpdateTime: metav1.NewTime(now),
},
},
{
name: "deployment exists, configmap is valid, exec into agent pod returns invalid JSON",
pinnipedObjects: []runtime.Object{
initialCredentialIssuer,
},
kubeObjects: []runtime.Object{
healthyKubeControllerManagerPod,
healthyAgentDeployment,
healthyAgentPod,
validClusterInfoConfigMap,
},
mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
Return("bogus-data", nil).
AnyTimes()
},
wantDistinctErrors: []string{
`failed to decode signing cert/key JSON from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: invalid character 'b' looking for beginning of value`,
},
wantAgentDeployment: healthyAgentDeployment,
wantStrategy: &configv1alpha1.CredentialIssuerStrategy{
Type: configv1alpha1.KubeClusterSigningCertificateStrategyType,
Status: configv1alpha1.ErrorStrategyStatus,
Reason: configv1alpha1.CouldNotFetchKeyStrategyReason,
Message: `failed to decode signing cert/key JSON from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: invalid character 'b' looking for beginning of value`,
LastUpdateTime: metav1.NewTime(now),
},
},
{
name: "deployment exists, configmap is valid, exec into agent pod returns invalid cert base64",
pinnipedObjects: []runtime.Object{
initialCredentialIssuer,
},
kubeObjects: []runtime.Object{
healthyKubeControllerManagerPod,
healthyAgentDeployment,
healthyAgentPod,
validClusterInfoConfigMap,
},
mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
Return(`{"tls.crt": "invalid"}`, nil).
AnyTimes()
},
wantDistinctErrors: []string{
`failed to decode signing cert base64 from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: illegal base64 data at input byte 4`,
},
wantAgentDeployment: healthyAgentDeployment,
wantStrategy: &configv1alpha1.CredentialIssuerStrategy{
Type: configv1alpha1.KubeClusterSigningCertificateStrategyType,
Status: configv1alpha1.ErrorStrategyStatus,
Reason: configv1alpha1.CouldNotFetchKeyStrategyReason,
Message: `failed to decode signing cert base64 from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: illegal base64 data at input byte 4`,
LastUpdateTime: metav1.NewTime(now),
},
},
{
name: "deployment exists, configmap is valid, exec into agent pod returns invalid key base64",
pinnipedObjects: []runtime.Object{
initialCredentialIssuer,
},
kubeObjects: []runtime.Object{
healthyKubeControllerManagerPod,
healthyAgentDeployment,
healthyAgentPod,
validClusterInfoConfigMap,
},
mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
Return(`{"tls.crt": "dGVzdAo=", "tls.key": "invalid"}`, nil).
AnyTimes()
},
wantDistinctErrors: []string{
`failed to decode signing key base64 from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: illegal base64 data at input byte 4`,
},
wantAgentDeployment: healthyAgentDeployment,
wantStrategy: &configv1alpha1.CredentialIssuerStrategy{
Type: configv1alpha1.KubeClusterSigningCertificateStrategyType,
Status: configv1alpha1.ErrorStrategyStatus,
Reason: configv1alpha1.CouldNotFetchKeyStrategyReason,
Message: `failed to decode signing key base64 from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: illegal base64 data at input byte 4`,
LastUpdateTime: metav1.NewTime(now),
},
},
{
name: "deployment exists, configmap is valid, exec into agent pod returns bogus certs",
pinnipedObjects: []runtime.Object{
@@ -601,10 +685,10 @@ func TestAgentController(t *testing.T) {
validClusterInfoConfigMap,
},
mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}").
Return("bogus-data", nil).
executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
Return(`{"tls.crt": "dGVzdC1jZXJ0", "tls.key": "dGVzdC1rZXk="}`, nil). // "test-cert" / "test-key"
AnyTimes()
dynamicCert.SetCertKeyContent([]byte(""), []byte("")).
dynamicCert.SetCertKeyContent([]byte("test-cert"), []byte("test-key")).
Return(fmt.Errorf("some dynamic cert error")).
AnyTimes()
},