Switch back to an exec-based approach to grab the controller-manager CA. (#65)

This switches us back to an approach where we use the Pod "exec" API to grab the keys we need, rather than forcing our code to run on the control plane node. It will help us fail gracefully (or dynamically switch to alternate implementations) when the cluster is not self-hosted.

Signed-off-by: Matt Moyer <moyerm@vmware.com>
Co-authored-by: Ryan Richard <richardry@vmware.com>

deploy/deployment.yaml

@@ -46,13 +46,14 @@ data:
 #! TODO set up healthy, ready, etc. probes correctly?
 #! TODO set resource minimums (e.g. 512MB RAM) to make sure we get scheduled onto a reasonable node?
 apiVersion: apps/v1
-kind: DaemonSet
+kind: Deployment
 metadata:
   name: #@ data.values.app_name
   namespace: #@ data.values.namespace
   labels:
     app: #@ data.values.app_name
 spec:
+  replicas: 2
   selector:
     matchLabels:
       app: #@ data.values.app_name
@@ -79,15 +80,11 @@ spec:
           args:
             - --config=/etc/config/placeholder-name.yaml
             - --downward-api-path=/etc/podinfo
-            - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
-            - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
           volumeMounts:
             - name: config-volume
               mountPath: /etc/config
             - name: podinfo
               mountPath: /etc/podinfo
-            - name: k8s-certs
-              mountPath: /etc/kubernetes/pki
           livenessProbe:
             httpGet:
               path: /healthz
@@ -119,16 +116,10 @@ spec:
               - path: "namespace"
                 fieldRef:
                   fieldPath: metadata.namespace
-        - name: k8s-certs
-          hostPath:
-            path: /etc/kubernetes/pki
-            type: DirectoryOrCreate
-      nodeSelector: #! Create Pods on all nodes which match this node selector, and not on any other nodes.
-        node-role.kubernetes.io/master: ""
       tolerations:
         - key: CriticalAddonsOnly
           operator: Exists
-        - key: node-role.kubernetes.io/master #! Allow running on master nodes.
+        - key: node-role.kubernetes.io/master #! Allow running on master nodes too
           effect: NoSchedule
       #! "system-cluster-critical" cannot be used outside the kube-system namespace until Kubernetes >= 1.17,
       #! so we skip setting this for now (see https://github.com/kubernetes/kubernetes/issues/60596).

deploy/rbac.yaml

@@ -62,6 +62,35 @@ roleRef:
   name: #@ data.values.app_name + "-aggregated-api-server-role"
   apiGroup: rbac.authorization.k8s.io
 
+#! Give permission to list pods and pod exec in the kube-system namespace so we can find the API server's private key
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: #@ data.values.app_name + "-kube-system-pod-exec-role"
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources: [pods]
+    verbs: [get, list]
+  - apiGroups: [""]
+    resources: [pods/exec]
+    verbs: [create]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: #@ data.values.app_name + "-kube-system-pod-exec-role-binding"
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: #@ data.values.app_name + "-service-account"
+    namespace: #@ data.values.namespace
+roleRef:
+  kind: Role
+  name: #@ data.values.app_name + "-kube-system-pod-exec-role"
+  apiGroup: rbac.authorization.k8s.io
+
 #! Allow both authenticated and unauthenticated CredentialRequests (i.e. allow all requests)
 ---
 apiVersion: rbac.authorization.k8s.io/v1