From d0492fdee5af9d7197b6142d645f957ed6da2b04 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Tue, 16 Jan 2024 12:48:15 +0000 Subject: [PATCH] cmd/k8s-operator: adds a tailscale IngressClass resource, prints warning if class not found. (#10823) * cmd/k8s-operator/deploy: deploy a Tailscale IngressClass resource. Some Ingress validating webhooks reject Ingresses with .spec.ingressClassName for which there is no matching IngressClass. Additionally, validate that the expected IngressClass is present, when parsing a tailscale `Ingress`. We currently do not utilize the IngressClass, however we might in the future at which point we might start requiring that the right class for this controller instance actually exists. Updates tailscale/tailscale#10820 Signed-off-by: Irbe Krumina Co-authored-by: Anton Tolchanov --- .../deploy/chart/templates/ingressclass.yaml | 8 ++++ .../deploy/chart/templates/operator-rbac.yaml | 3 ++ .../deploy/manifests/operator.yaml | 16 ++++++++ cmd/k8s-operator/ingress.go | 37 ++++++++++++++++++- cmd/k8s-operator/operator.go | 3 ++ 5 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml diff --git a/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml b/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml new file mode 100644 index 000000000..2a1fa81b4 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml @@ -0,0 +1,8 @@ +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: tailscale # class name currently can not be changed + annotations: {} # we do not support default IngressClass annotation https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class +spec: + controller: tailscale.com/ts-ingress # controller name currently can not be changed + # parameters: {} # currently no parameters are supported diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index fbd83e7e1..8ea07e808 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -18,6 +18,9 @@ rules: - apiGroups: ["networking.k8s.io"] resources: ["ingresses", "ingresses/status"] verbs: ["*"] +- apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] - apiGroups: ["tailscale.com"] resources: ["connectors", "connectors/status"] verbs: ["get", "list", "watch", "update"] diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index afdf47135..151eec620 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -173,6 +173,14 @@ rules: - ingresses/status verbs: - '*' + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch - apiGroups: - tailscale.com resources: @@ -312,3 +320,11 @@ spec: - name: oauth secret: secretName: operator-oauth +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + annotations: {} + name: tailscale +spec: + controller: tailscale.com/ts-ingress diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 0c306fc52..38667c1cd 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -12,10 +12,12 @@ import ( "strings" "sync" + "github.com/pkg/errors" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -26,6 +28,12 @@ import ( "tailscale.com/util/set" ) +const ( + tailscaleIngressClassName = "tailscale" // ingressClass.metadata.name for tailscale IngressClass resource + tailscaleIngressControllerName = "tailscale.com/ts-ingress" // ingressClass.spec.controllerName for tailscale IngressClass resource + ingressClassDefaultAnnotation = "ingressclass.kubernetes.io/is-default-class" // we do not support this https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class +) + type IngressReconciler struct { client.Client @@ -109,6 +117,10 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare // This function adds a finalizer to ing, ensuring that we can handle orderly // deprovisioning later. func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error { + if err := a.validateIngressClass(ctx); err != nil { + logger.Warnf("error validating tailscale IngressClass: %v. In future this might be a terminal error.", err) + + } if !slices.Contains(ing.Finalizers, FinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, @@ -267,5 +279,28 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { return ing != nil && ing.Spec.IngressClassName != nil && - *ing.Spec.IngressClassName == "tailscale" + *ing.Spec.IngressClassName == tailscaleIngressClassName +} + +// validateIngressClass attempts to validate that 'tailscale' IngressClass +// included in Tailscale installation manifests exists and has not been modified +// to attempt to enable features that we do not support. +func (a *IngressReconciler) validateIngressClass(ctx context.Context) error { + ic := &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: tailscaleIngressClassName, + }, + } + if err := a.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) { + return errors.New("Tailscale IngressClass not found in cluster. Latest installation manifests include a tailscale IngressClass - please update") + } else if err != nil { + return fmt.Errorf("error retrieving 'tailscale' IngressClass: %w", err) + } + if ic.Spec.Controller != tailscaleIngressControllerName { + return fmt.Errorf("Tailscale Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName) + } + if ic.GetAnnotations()[ingressClassDefaultAnnotation] != "" { + return fmt.Errorf("%s annotation is set on 'tailscale' IngressClass, but Tailscale Ingress controller does not support default Ingress class. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ingressClassDefaultAnnotation) + } + return nil } diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index c65ada481..483a88bba 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -215,6 +215,9 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string Field: client.InNamespace(tsNamespace).AsSelector(), } mgrOpts := manager.Options{ + // TODO (irbekrm): stricter filtering what we watch/cache/call + // reconcilers on. c/r by default starts a watch on any + // resources that we GET via the controller manager's client. Cache: cache.Options{ ByObject: map[client.Object]cache.ByObject{ &corev1.Secret{}: nsFilter,