diff --git a/cmd/k8s-operator/manifests/operator.yaml b/cmd/k8s-operator/manifests/operator.yaml index 246ff330b..e2f98c146 100644 --- a/cmd/k8s-operator/manifests/operator.yaml +++ b/cmd/k8s-operator/manifests/operator.yaml @@ -153,6 +153,8 @@ spec: value: tag:k8s - name: APISERVER_PROXY value: "false" + - name: PROXY_FIREWALL_MODE + value: auto volumeMounts: - name: oauth mountPath: /oauth diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index b5f11e5a6..a5d6cb07f 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -52,6 +52,7 @@ func main() { image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") tags = defaultEnv("PROXY_TAGS", "tag:k8s") + tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") ) var opts []kzap.Opts @@ -70,7 +71,7 @@ func main() { defer s.Close() restConfig := config.GetConfigOrDie() maybeLaunchAPIServerProxy(zlog, restConfig, s) - runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags) + runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode) } // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the @@ -179,7 +180,7 @@ waitOnline: // runReconcilers starts the controller-runtime manager and registers the // ServiceReconciler. It blocks forever. -func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags string) { +func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string) { var ( isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) ) @@ -216,6 +217,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string operatorNamespace: tsNamespace, proxyImage: image, proxyPriorityClassName: priorityClassName, + tsFirewallMode: tsFirewallMode, } err = builder. ControllerManagedBy(mgr). @@ -228,6 +230,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string Client: mgr.GetClient(), logger: zlog.Named("service-reconciler"), isDefaultLoadBalancer: isDefaultLoadBalancer, + recorder: eventRecorder, }) if err != nil { startlog.Fatalf("could not create controller: %v", err) diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index b251d81bf..985a29fa7 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -70,7 +70,12 @@ func TestLoadBalancerClass(t *testing.T) { expectEqual(t, fc, expectedSecret(fullName)) expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", "")) + o := stsOpts{ + name: shortName, + secretName: fullName, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -202,7 +207,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(fullName)) expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", "")) + o := stsOpts{ + name: shortName, + secretName: fullName, + tailnetTargetIP: tailnetTargetIP, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -226,7 +237,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(fullName)) expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", "")) + o = stsOpts{ + name: shortName, + secretName: fullName, + tailnetTargetIP: tailnetTargetIP, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) // Change the tailscale-target-ip annotation which should update the // StatefulSet @@ -305,7 +322,12 @@ func TestAnnotations(t *testing.T) { expectEqual(t, fc, expectedSecret(fullName)) expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", "")) + o := stsOpts{ + name: shortName, + secretName: fullName, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -405,7 +427,12 @@ func TestAnnotationIntoLB(t *testing.T) { expectEqual(t, fc, expectedSecret(fullName)) expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", "")) + o := stsOpts{ + name: shortName, + secretName: fullName, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, since it would have normally happened at @@ -450,7 +477,12 @@ func TestAnnotationIntoLB(t *testing.T) { expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", "")) + o = stsOpts{ + name: shortName, + secretName: fullName, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) // ... but the service should have a LoadBalancer status. want = &corev1.Service{ @@ -528,7 +560,12 @@ func TestLBIntoAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(fullName)) expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", "")) + o := stsOpts{ + name: shortName, + secretName: fullName, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -591,7 +628,12 @@ func TestLBIntoAnnotation(t *testing.T) { expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", "")) + o = stsOpts{ + name: shortName, + secretName: fullName, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) want = &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -661,7 +703,12 @@ func TestCustomHostname(t *testing.T) { expectEqual(t, fc, expectedSecret(fullName)) expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedSTS(shortName, fullName, "reindeer-flotilla", "")) + o := stsOpts{ + name: shortName, + secretName: fullName, + hostname: "reindeer-flotilla", + } + expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -735,7 +782,7 @@ func TestCustomPriorityClassName(t *testing.T) { defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", - proxyPriorityClassName: "tailscale-critical", + proxyPriorityClassName: "custom-priority-class-name", }, logger: zl.Sugar(), } @@ -752,7 +799,7 @@ func TestCustomPriorityClassName(t *testing.T) { UID: types.UID("1234-UID"), Annotations: map[string]string{ "tailscale.com/expose": "true", - "tailscale.com/hostname": "custom-priority-class-name", + "tailscale.com/hostname": "tailscale-critical", }, }, Spec: corev1.ServiceSpec{ @@ -764,8 +811,14 @@ func TestCustomPriorityClassName(t *testing.T) { expectReconciled(t, sr, "default", "test") fullName, shortName := findGenName(t, fc, "default", "test") + o := stsOpts{ + name: shortName, + secretName: fullName, + hostname: "tailscale-critical", + priorityClassName: "custom-priority-class-name", + } - expectEqual(t, fc, expectedSTS(shortName, fullName, "custom-priority-class-name", "tailscale-critical")) + expectEqual(t, fc, expectedSTS(o)) } func TestDefaultLoadBalancer(t *testing.T) { @@ -811,7 +864,63 @@ func TestDefaultLoadBalancer(t *testing.T) { expectEqual(t, fc, expectedSecret(fullName)) expectEqual(t, fc, expectedHeadlessService(shortName)) - expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", "")) + o := stsOpts{ + name: shortName, + secretName: fullName, + hostname: "default-test", + } + expectEqual(t, fc, expectedSTS(o)) +} + +func TestProxyFirewallMode(t *testing.T) { + fc := fake.NewFakeClient() + ft := &fakeTSClient{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + tsFirewallMode: "nftables", + }, + logger: zl.Sugar(), + isDefaultLoadBalancer: true, + } + + // Create a service that we should manage, and check that the initial round + // of objects looks right. + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + }, + }) + + expectReconciled(t, sr, "default", "test") + + fullName, shortName := findGenName(t, fc, "default", "test") + o := stsOpts{ + name: shortName, + secretName: fullName, + hostname: "default-test", + firewallMode: "nftables", + } + expectEqual(t, fc, expectedSTS(o)) + } func expectedSecret(name string) *corev1.Secret { @@ -862,83 +971,44 @@ func expectedHeadlessService(name string) *corev1.Service { } } -func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv1.StatefulSet { - return &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - Kind: "StatefulSet", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: stsName, - Namespace: "operator-ns", - Labels: map[string]string{ - "tailscale.com/managed": "true", - "tailscale.com/parent-resource": "test", - "tailscale.com/parent-resource-ns": "default", - "tailscale.com/parent-resource-type": "svc", - }, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: ptr.To[int32](1), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "1234-UID"}, - }, - ServiceName: stsName, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "tailscale.com/operator-last-set-hostname": hostname, - "tailscale.com/operator-last-set-cluster-ip": "10.20.30.40", - }, - DeletionGracePeriodSeconds: ptr.To[int64](10), - Labels: map[string]string{"app": "1234-UID"}, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: "proxies", - PriorityClassName: priorityClassName, - InitContainers: []corev1.Container{ - { - Name: "sysctler", - Image: "tailscale/tailscale", - Command: []string{"/bin/sh"}, - Args: []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"}, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), - }, - }, - }, - Containers: []corev1.Container{ - { - Name: "tailscale", - Image: "tailscale/tailscale", - Env: []corev1.EnvVar{ - {Name: "TS_USERSPACE", Value: "false"}, - {Name: "TS_AUTH_ONCE", Value: "true"}, - {Name: "TS_KUBE_SECRET", Value: secretName}, - {Name: "TS_HOSTNAME", Value: hostname}, - {Name: "TS_DEST_IP", Value: "10.20.30.40"}, - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN"}, - }, - }, - ImagePullPolicy: "Always", - }, - }, - }, - }, - }, +func expectedSTS(opts stsOpts) *appsv1.StatefulSet { + containerEnv := []corev1.EnvVar{ + {Name: "TS_USERSPACE", Value: "false"}, + {Name: "TS_AUTH_ONCE", Value: "true"}, + {Name: "TS_KUBE_SECRET", Value: opts.secretName}, + {Name: "TS_HOSTNAME", Value: opts.hostname}, + } + annots := map[string]string{ + "tailscale.com/operator-last-set-hostname": opts.hostname, + } + if opts.tailnetTargetIP != "" { + annots["tailscale.com/operator-last-set-ts-tailnet-target-ip"] = opts.tailnetTargetIP + containerEnv = append(containerEnv, corev1.EnvVar{ + Name: "TS_TAILNET_TARGET_IP", + Value: opts.tailnetTargetIP, + }) + } else { + containerEnv = append(containerEnv, corev1.EnvVar{ + Name: "TS_DEST_IP", + Value: "10.20.30.40", + }) + + annots["tailscale.com/operator-last-set-cluster-ip"] = "10.20.30.40" + + } + if opts.firewallMode != "" { + containerEnv = append(containerEnv, corev1.EnvVar{ + Name: "TS_DEBUG_FIREWALL_MODE", + Value: opts.firewallMode, + }) } -} -func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityClassName string) *appsv1.StatefulSet { return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: stsName, + Name: opts.name, Namespace: "operator-ns", Labels: map[string]string{ "tailscale.com/managed": "true", @@ -952,19 +1022,16 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": "1234-UID"}, }, - ServiceName: stsName, + ServiceName: opts.name, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "tailscale.com/operator-last-set-hostname": hostname, - "tailscale.com/operator-last-set-ts-tailnet-target-ip": tailnetTargetIP, - }, + Annotations: annots, DeletionGracePeriodSeconds: ptr.To[int64](10), Labels: map[string]string{"app": "1234-UID"}, }, Spec: corev1.PodSpec{ ServiceAccountName: "proxies", - PriorityClassName: priorityClassName, + PriorityClassName: opts.priorityClassName, InitContainers: []corev1.Container{ { Name: "sysctler", @@ -980,13 +1047,7 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC { Name: "tailscale", Image: "tailscale/tailscale", - Env: []corev1.EnvVar{ - {Name: "TS_USERSPACE", Value: "false"}, - {Name: "TS_AUTH_ONCE", Value: "true"}, - {Name: "TS_KUBE_SECRET", Value: secretName}, - {Name: "TS_HOSTNAME", Value: hostname}, - {Name: "TS_TAILNET_TARGET_IP", Value: tailnetTargetIP}, - }, + Env: containerEnv, SecurityContext: &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ Add: []corev1.Capability{"NET_ADMIN"}, @@ -1126,6 +1187,15 @@ func expectRequeue(t *testing.T, sr *ServiceReconciler, ns, name string) { } } +type stsOpts struct { + name string + secretName string + hostname string + priorityClassName string + firewallMode string + tailnetTargetIP string +} + type fakeTSClient struct { sync.Mutex keyRequests []tailscale.KeyCapabilities diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index bc1bea137..b4a694908 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -79,6 +79,14 @@ type tailscaleSTSReconciler struct { operatorNamespace string proxyImage string proxyPriorityClassName string + tsFirewallMode string +} + +func (sts tailscaleSTSReconciler) validate() error { + if sts.tsFirewallMode != "" && !isValidFirewallMode(sts.tsFirewallMode) { + return fmt.Errorf("invalid proxy firewall mode %s, valid modes are iptables, nftables or unset", sts.tsFirewallMode) + } + return nil } // IsHTTPSEnabledOnTailnet reports whether HTTPS is enabled on the tailnet. @@ -360,6 +368,13 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }, }) } + if a.tsFirewallMode != "" { + container.Env = append(container.Env, corev1.EnvVar{ + Name: "TS_DEBUG_FIREWALL_MODE", + Value: a.tsFirewallMode, + }, + ) + } ss.ObjectMeta = metav1.ObjectMeta{ Name: headlessSvc.Name, Namespace: a.operatorNamespace, @@ -499,3 +514,7 @@ func nameForService(svc *corev1.Service) (string, error) { } return svc.Namespace + "-" + svc.Name, nil } + +func isValidFirewallMode(m string) bool { + return m == "auto" || m == "nftables" || m == "iptables" +} diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index 701c3fb18..b927cda4f 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -17,6 +17,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/util/clientmetric" @@ -37,6 +38,8 @@ type ServiceReconciler struct { // managedEgressProxies is a set of all egress proxies that we're currently // managing. This is only used for metrics. managedEgressProxies set.Slice[types.UID] + + recorder record.EventRecorder } var ( @@ -136,6 +139,15 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare // This function adds a finalizer to svc, ensuring that we can handle orderly // deprovisioning later. func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error { + // run for proxy config related validations here as opposed to running + // them earlier. This is to prevent cleanup etc being blocked on a + // misconfigured proxy param + if err := a.ssr.validate(); err != nil { + msg := fmt.Sprintf("unable to provision proxy resources: invalid config: %v", err) + a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDCONFIG", msg) + a.logger.Error(msg) + return nil + } hostname, err := nameForService(svc) if err != nil { return err