From ed1b935238a7c6cce3aa4865aa13181eb679361a Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 30 Oct 2023 18:18:09 +0000 Subject: [PATCH] cmd/k8s-operator: allow to install operator via helm (#9920) Initial helm manifests. Updates tailscale/tailscale#9222 Signed-off-by: Irbe Krumina Co-authored-by: Maisem Ali --- cmd/k8s-operator/deploy/chart/.helmignore | 23 +++++ cmd/k8s-operator/deploy/chart/Chart.yaml | 29 ++++++ .../chart/templates/apiserverproxy-rbac.yaml | 26 ++++++ .../deploy/chart/templates/deployment.yaml | 90 +++++++++++++++++++ .../deploy/chart/templates/oauth-secret.yaml | 13 +++ .../deploy/chart/templates/operator-rbac.yaml | 60 +++++++++++++ .../deploy/chart/templates/proxy-rbac.yaml | 32 +++++++ cmd/k8s-operator/deploy/chart/values.yaml | 45 ++++++++++ .../manifests/authproxy-rbac.yaml | 0 .../{ => deploy}/manifests/operator.yaml | 0 .../{ => deploy}/manifests/proxy.yaml | 0 .../manifests/userspace-proxy.yaml | 0 cmd/k8s-operator/sts.go | 4 +- 13 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 cmd/k8s-operator/deploy/chart/.helmignore create mode 100644 cmd/k8s-operator/deploy/chart/Chart.yaml create mode 100644 cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml create mode 100644 cmd/k8s-operator/deploy/chart/templates/deployment.yaml create mode 100644 cmd/k8s-operator/deploy/chart/templates/oauth-secret.yaml create mode 100644 cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml create mode 100644 cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml create mode 100644 cmd/k8s-operator/deploy/chart/values.yaml rename cmd/k8s-operator/{ => deploy}/manifests/authproxy-rbac.yaml (100%) rename cmd/k8s-operator/{ => deploy}/manifests/operator.yaml (100%) rename cmd/k8s-operator/{ => deploy}/manifests/proxy.yaml (100%) rename cmd/k8s-operator/{ => deploy}/manifests/userspace-proxy.yaml (100%) diff --git a/cmd/k8s-operator/deploy/chart/.helmignore b/cmd/k8s-operator/deploy/chart/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/cmd/k8s-operator/deploy/chart/Chart.yaml b/cmd/k8s-operator/deploy/chart/Chart.yaml new file mode 100644 index 000000000..278e8f871 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/Chart.yaml @@ -0,0 +1,29 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + +apiVersion: v1 +name: tailscale-operator +description: A Helm chart for Tailscale Kubernetes operator +home: https://github.com/tailscale/tailscale + +keywords: + - "tailscale" + - "vpn" + - "ingress" + - "egress" + - "wireguard" + +sources: +- https://github.com/tailscale/tailscale + +type: application + +maintainers: + - name: tailscale-maintainers + url: https://tailscale.com/ + +# version will be set to Tailscale repo tag (without 'v') at release time. +version: 0.1.0 + +# appVersion will be set to Tailscale repo tag at release time. +appVersion: "unstable" diff --git a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml new file mode 100644 index 000000000..072ecf6d2 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml @@ -0,0 +1,26 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + +{{ if eq .Values.apiServerProxyConfig.mode "true" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tailscale-auth-proxy +rules: +- apiGroups: [""] + resources: ["users", "groups"] + verbs: ["impersonate"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tailscale-auth-proxy +subjects: +- kind: ServiceAccount + name: operator + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: tailscale-auth-proxy + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml new file mode 100644 index 000000000..fc39baff1 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml @@ -0,0 +1,90 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operator + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: operator + template: + metadata: + {{- with .Values.operatorConfig.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: operator + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: operator + {{- with .Values.operatorConfig.podSecurityContext }} + securityContext: + {{- toYaml .Values.operatorConfig.podSecurityContext | nindent 8 }} + {{- end }} + volumes: + - name: oauth + secret: + secretName: operator-oauth + containers: + - name: operator + {{- with .Values.operatorConfig.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.operatorConfig.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- $operatorTag:= printf ":%s" ( .Values.operatorConfig.image.tag | default .Chart.AppVersion )}} + image: {{ .Values.operatorConfig.image.repo }}{{- if .Values.operatorConfig.image.digest -}}{{ printf "@%s" .Values.operatorConfig.image.digest}}{{- else -}}{{ printf "%s" $operatorTag }}{{- end }} + imagePullPolicy: {{ .Values.operatorConfig.image.pullPolicy }} + env: + - name: OPERATOR_HOSTNAME + value: {{ .Values.operatorConfig.hostname }} + - name: OPERATOR_SECRET + value: operator + - name: OPERATOR_LOGGING + value: {{ .Values.operatorConfig.logging }} + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLIENT_ID_FILE + value: /oauth/client_id + - name: CLIENT_SECRET_FILE + value: /oauth/client_secret + {{- $proxyTag := printf ":%s" ( .Values.proxyConfig.image.tag | default .Chart.AppVersion )}} + - name: PROXY_IMAGE + value: {{ .Values.proxyConfig.image.repo }}{{- if .Values.proxyConfig.image.digest -}}{{ printf "@%s" .Values.proxyConfig.image.digest}}{{- else -}}{{ printf "%s" $proxyTag }}{{- end }} + - name: PROXY_TAGS + value: {{ .Values.proxyConfig.defaultTags }} + - name: APISERVER_PROXY + value: "{{ .Values.apiServerProxyConfig.mode }}" + - name: PROXY_FIREWALL_MODE + value: {{ .Values.proxyConfig.firewallMode }} + volumeMounts: + - name: oauth + mountPath: /oauth + readOnly: true + {{- with .Values.operatorConfig.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operatorConfig.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operatorConfig.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/cmd/k8s-operator/deploy/chart/templates/oauth-secret.yaml b/cmd/k8s-operator/deploy/chart/templates/oauth-secret.yaml new file mode 100644 index 000000000..b44fde0a1 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/oauth-secret.yaml @@ -0,0 +1,13 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + +{{ if and .Values.oauth .Values.oauth.clientId -}} +apiVersion: v1 +kind: Secret +metadata: + name: operator-oauth + namespace: {{ .Release.Namespace }} +stringData: + client_id: {{ .Values.oauth.clientId }} + client_secret: {{ .Values.oauth.clientSecret }} +{{- end -}} diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml new file mode 100644 index 000000000..5fc7ad203 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -0,0 +1,60 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tailscale-operator +rules: +- apiGroups: [""] + resources: ["events", "services", "services/status"] + verbs: ["*"] +- apiGroups: ["networking.k8s.io"] + resources: ["ingresses", "ingresses/status"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tailscale-operator +subjects: +- kind: ServiceAccount + name: operator + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: tailscale-operator + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["*"] +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operator + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: operator + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: operator + apiGroup: rbac.authorization.k8s.io diff --git a/cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml new file mode 100644 index 000000000..31a034aaa --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml @@ -0,0 +1,32 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxies + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: proxies + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: proxies + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: proxies + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: proxies + apiGroup: rbac.authorization.k8s.io diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml new file mode 100644 index 000000000..893e6c395 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -0,0 +1,45 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + +# Operator oauth credentials. If set a Kubernetes Secret with the provided +# values will be created in the operator namespace. If unset a Secret named +# operator-oauth must be precreated. +# oauth: +# clientId: "" +# clientSecret: "" + +operatorConfig: + image: + repo: tailscale/k8s-operator + # Digest will be prioritized over tag. If neither are set appVersion will be + # used. + tag: "" + digest: "" + logging: "info" + hostname: "tailscale-operator" + nodeSelector: + kubernetes.io/os: linux + + +# proxyConfig contains configuraton that will be applied to any ingress/egress +# proxies created by the operator. +# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-ingress +# https://tailscale.com/kb/1236/kubernetes-operator/#cluster-egress +proxyConfig: + image: + repo: tailscale/tailscale + # Digest will be prioritized over tag. If neither are set appVersion will be + # used. + tag: "" + digest: "" + # ACL tag that operator will tag proxies with. Operator must be made owner of + # these tags + # https://tailscale.com/kb/1236/kubernetes-operator/?q=operator#setting-up-the-kubernetes-operator + defaultTags: tag:k8s + firewallMode: auto + +# apiServerProxyConfig allows to configure whether the operator should expose +# Kubernetes API server. +# https://tailscale.com/kb/1236/kubernetes-operator/#accessing-the-kubernetes-control-plane-using-an-api-server-proxy +apiServerProxyConfig: + mode: "false" # "true", "false", "noauth" diff --git a/cmd/k8s-operator/manifests/authproxy-rbac.yaml b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml similarity index 100% rename from cmd/k8s-operator/manifests/authproxy-rbac.yaml rename to cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml diff --git a/cmd/k8s-operator/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml similarity index 100% rename from cmd/k8s-operator/manifests/operator.yaml rename to cmd/k8s-operator/deploy/manifests/operator.yaml diff --git a/cmd/k8s-operator/manifests/proxy.yaml b/cmd/k8s-operator/deploy/manifests/proxy.yaml similarity index 100% rename from cmd/k8s-operator/manifests/proxy.yaml rename to cmd/k8s-operator/deploy/manifests/proxy.yaml diff --git a/cmd/k8s-operator/manifests/userspace-proxy.yaml b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml similarity index 100% rename from cmd/k8s-operator/manifests/userspace-proxy.yaml rename to cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 5e26f3de3..d133b8bab 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -307,10 +307,10 @@ func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string) return key, nil } -//go:embed manifests/proxy.yaml +//go:embed deploy/manifests/proxy.yaml var proxyYaml []byte -//go:embed manifests/userspace-proxy.yaml +//go:embed deploy/manifests/userspace-proxy.yaml var userspaceProxyYaml []byte func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, authKeySecret string) (*appsv1.StatefulSet, error) {