diff --git a/Makefile b/Makefile index 80bd6fe88..e50e4c759 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ PWD := $(shell pwd) GOPATH := $(shell go env GOPATH) +# Sets the build version based on the output of the following command, if we are building for a tag, that's the build else it uses the current git branch as the build +BUILD_VERSION:=$(shell git describe --exact-match --tags $(git log -n1 --pretty='%h') 2>/dev/null || git rev-parse --abbrev-ref HEAD 2>/dev/null) +BUILD_TIME:=$(shell date 2>/dev/null) +TAG ?= "minio/m3:$(VERSION)-dev" default: mcs @@ -8,6 +12,11 @@ mcs: @echo "Building mcs binary to './mcs'" @(GO111MODULE=on CGO_ENABLED=0 go build -trimpath --tags=kqueue --ldflags "-s -w" -o mcs ./cmd/mcs) +k8sdev: + @docker build -t $(TAG) --build-arg build_version=$(BUILD_VERSION) --build-arg build_time='$(BUILD_TIME)' . + @kind load docker-image $(TAG) + @echo "Done, now restart your mcs deployment" + getdeps: @mkdir -p ${GOPATH}/bin @which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0) diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 000000000..68c59822b --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# beginning of customizations +resources: + - mcs-service-account.yaml + - mcs-cluster-role.yaml + - mcs-cluster-role-binding.yaml + - mcs-configmap.yaml + - mcs-service.yaml + - mcs-deployment.yaml + - minio-operator.yaml diff --git a/k8s/base/mcs-cluster-role-binding.yaml b/k8s/base/mcs-cluster-role-binding.yaml new file mode 100644 index 000000000..3eb272c8c --- /dev/null +++ b/k8s/base/mcs-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mcs-sa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mcs-sa-role +subjects: + - kind: ServiceAccount + name: mcs-sa + namespace: default diff --git a/k8s/base/mcs-cluster-role.yaml b/k8s/base/mcs-cluster-role.yaml new file mode 100644 index 000000000..e22d7059d --- /dev/null +++ b/k8s/base/mcs-cluster-role.yaml @@ -0,0 +1,77 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mcs-sa-role +rules: + - apiGroups: + - "" + resources: + - namespaces + - secrets + - pods + - services + - events + - resourcequotas + verbs: + - get + - watch + - create + - list + - patch + - apiGroups: + - "storage.k8s.io" + resources: + - storageclasses + verbs: + - get + - watch + - create + - list + - patch + - apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - get + - create + - list + - patch + - watch + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - create + - list + - patch + - watch + - update + - delete + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + - "certificatesigningrequests/approval" + - "certificatesigningrequests/status" + verbs: + - update + - create + - get + - apiGroups: + - operator.min.io + resources: + - "*" + verbs: + - "*" + - apiGroups: + - min.io + resources: + - "*" + verbs: + - "*" diff --git a/k8s/base/mcs-configmap.yaml b/k8s/base/mcs-configmap.yaml new file mode 100644 index 000000000..e2da243b8 --- /dev/null +++ b/k8s/base/mcs-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mcs-env +data: + MCS_PORT: "9090" + MCS_TLS_PORT: "9443" diff --git a/k8s/base/mcs-deployment.yaml b/k8s/base/mcs-deployment.yaml new file mode 100644 index 000000000..e67a50bbb --- /dev/null +++ b/k8s/base/mcs-deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mcs +spec: + replicas: 1 + selector: + matchLabels: + app: mcs + template: + metadata: + labels: + app: mcs + spec: + serviceAccountName: m3-sa + containers: + - name: mcs + image: minio/mcs:latest + imagePullPolicy: "IfNotPresent" + env: + - name: MCS_MKUBE_ADMIN_ONLY + value: "on" + args: + - /mcs + - server + ports: + - containerPort: 9090 + name: http + - containerPort: 9433 + name: https diff --git a/k8s/base/mcs-service-account.yaml b/k8s/base/mcs-service-account.yaml new file mode 100644 index 000000000..1bde99a51 --- /dev/null +++ b/k8s/base/mcs-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mcs-sa + namespace: default diff --git a/k8s/base/mcs-service.yaml b/k8s/base/mcs-service.yaml new file mode 100644 index 000000000..8b5f0a5c4 --- /dev/null +++ b/k8s/base/mcs-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: mcs + labels: + name: mcs +spec: + ports: + - port: 9090 + name: http + - port: 9443 + name: https + selector: + app: mcs diff --git a/k8s/base/minio-operator.yaml b/k8s/base/minio-operator.yaml new file mode 100644 index 000000000..39e368fe5 --- /dev/null +++ b/k8s/base/minio-operator.yaml @@ -0,0 +1,203 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: minioinstances.operator.min.io +spec: + group: operator.min.io + scope: Namespaced + names: + kind: MinIOInstance + singular: minioinstance + plural: minioinstances + versions: + - name: v1 + served: true + storage: true + schema: + # openAPIV3Schema is the schema for validating custom objects. + # Refer https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#specifying-a-structural-schema + # for more details + openAPIV3Schema: + type: object + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + replicas: + type: integer + minimum: 1 + maximum: 32 + image: + type: string + serviceName: + type: string + volumesPerServer: + type: integer + mountPath: + type: string + podManagementPolicy: + type: string + enum: [Parallel, OrderedReady] + default: Parallel + requestAutoCert: + type: boolean + default: false + version: + type: string + mountpath: + type: string + subpath: + type: string + mcs: + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + image: + type: string + replicas: + type: integer + default: 2 + mcsSecret: + type: object + properties: + name: + type: string + kes: + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + image: + type: string + replicas: + type: integer + default: 2 + kesSecret: + type: object + properties: + name: + type: string + status: + type: object + properties: + currentState: + type: string + subresources: + # status enables the status subresource. + status: {} + additionalPrinterColumns: + - name: Current State + type: string + jsonPath: ".status.currentState" +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: minio-operator-role +rules: + - apiGroups: + - "" + resources: + - namespaces + - secrets + - pods + - services + - events + verbs: + - get + - watch + - create + - list + - delete + - apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - get + - create + - list + - patch + - watch + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - create + - list + - patch + - watch + - update + - delete + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + - "certificatesigningrequests/approval" + - "certificatesigningrequests/status" + verbs: + - update + - create + - get + - delete + - apiGroups: + - operator.min.io + resources: + - "*" + verbs: + - "*" + - apiGroups: + - min.io + resources: + - "*" + verbs: + - "*" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: minio-operator + namespace: default +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: minio-operator-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: minio-operator-role +subjects: + - kind: ServiceAccount + name: minio-operator + namespace: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio-operator + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + name: minio-operator + template: + metadata: + labels: + name: minio-operator + spec: + serviceAccountName: minio-operator + containers: + - name: minio-operator + image: minio/k8s-operator:2.0.8 + imagePullPolicy: IfNotPresent + # To specify cluster domain, un comment the following: + # env: + # - name: CLUSTER_DOMAIN + # value: mycluster.mydomain diff --git a/k8s/create-kind.sh b/k8s/create-kind.sh new file mode 100755 index 000000000..ae79b5aa6 --- /dev/null +++ b/k8s/create-kind.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# setup environment variables based on flags to see if we should build the docker containers again +MCS_DOCKER="true" + +# evaluate flags +# `-m` for mcs + + +while getopts ":m:" opt; do + case $opt in + m) + MCS_DOCKER="$OPTARG" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +echo "Provisioning Kind" +kind create cluster --config kind-cluster.yaml +echo "Remove Master Taint" +kubectl taint nodes --all node-role.kubernetes.io/master- +echo "Install Contour" +kubectl apply -f https://projectcontour.io/quickstart/contour.yaml +kubectl patch daemonsets -n projectcontour envoy -p '{"spec":{"template":{"spec":{"nodeSelector":{"ingress-ready":"true"},"tolerations":[{"key":"node-role.kubernetes.io/master","operator":"Equal","effect":"NoSchedule"}]}}}}' +echo "install metrics server" +kubectl apply -f metrics-dev.yaml + +# Whether or not to build the m3 container and load it to kind or just load it +if [[ $MCS_DOCKER == "true" ]]; then + # Build mkube + make --directory=".." k8sdev TAG=minio/mcs:latest +else + kind load docker-image minio/mcs:latest +fi + +echo "done" diff --git a/k8s/getoperator.sh b/k8s/getoperator.sh new file mode 100755 index 000000000..2a714e08e --- /dev/null +++ b/k8s/getoperator.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Get's the latest deployment file from MinIO Operator +curl https://raw.githubusercontent.com/minio/minio-operator/master/minio-operator.yaml > base/minio-operator.yaml diff --git a/k8s/kind-cluster.yaml b/k8s/kind-cluster.yaml new file mode 100644 index 000000000..c1529aa5d --- /dev/null +++ b/k8s/kind-cluster.yaml @@ -0,0 +1,22 @@ +# three node (two workers) cluster config +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 8844 + protocol: TCP + - containerPort: 443 + hostPort: 8843 + protocol: TCP +#- role: worker +#- role: worker +#- role: worker +#- role: worker diff --git a/k8s/metrics-dev.yaml b/k8s/metrics-dev.yaml new file mode 100644 index 000000000..b88b97d4c --- /dev/null +++ b/k8s/metrics-dev.yaml @@ -0,0 +1,153 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:aggregated-metrics-reader + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server +rules: + - apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + - namespaces + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: main-port +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server +spec: + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + containers: + - name: metrics-server + image: k8s.gcr.io/metrics-server-amd64:v0.3.6 + args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-insecure-tls + - --kubelet-preferred-address-types=InternalIP + ports: + - name: main-port + containerPort: 4443 + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + imagePullPolicy: Always + volumeMounts: + - name: tmp-dir + mountPath: /tmp + nodeSelector: + beta.kubernetes.io/os: linux + kubernetes.io/arch: "amd64"