mirrors/tailscale
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

cmd/k8s-operator,cmd/containerboot: add kube egress proxy (#9031)

Browse Source 
First part of work for the functionality that allows users to create an egress
proxy to access Tailnet services from within Kubernetes cluster workloads.
This PR allows creating an egress proxy that can access Tailscale services over HTTP only.

Updates tailscale/tailscale#8184

Signed-off-by: irbekrm <irbekrm@gmail.com>
Co-authored-by: Maisem Ali <maisem@tailscale.com>
Co-authored-by: Rhea Ghosh <rhea@tailscale.com>
This commit is contained in:
Irbe Krumina
2023-08-30 08:31:37 +01:00
committed by GitHub
parent ae747a2e48
commit fe709c81e5
5 changed files with 338 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/k8s-operator/ingress.go
View File

@ -190,7 +190,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
ChildResourceLabels: crl,
}
if err := a.ssr.Provision(ctx, logger, sts); err != nil {
if _, err := a.ssr.Provision(ctx, logger, sts); err != nil {
return fmt.Errorf("failed to provision: %w", err)
}

179
cmd/k8s-operator/operator_test.go
View File

@ -7,6 +7,7 @@ package main
import (
"context"
"fmt"
"strings"
"sync"
"testing"
@ -153,6 +154,111 @@ func TestLoadBalancerClass(t *testing.T) {
}
expectEqual(t, fc, want)
}
func TestTailnetTargetIPAnnotation(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
tailnetTargetIP := "100.66.66.66"
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
},
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: map[string]string{
"foo": "bar",
},
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test")
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
},
},
Spec: corev1.ServiceSpec{
ExternalName: fmt.Sprintf("%s.operator-ns.svc", shortName),
Type: corev1.ServiceTypeExternalName,
Selector: nil,
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(fullName))
expectEqual(t, fc, expectedHeadlessService(shortName))
expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
// Change the tailscale-target-ip annotation which should update the
// StatefulSet
tailnetTargetIP = "100.77.77.77"
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.ObjectMeta.Annotations = map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
}
})
// Remove the tailscale-target-ip annotation which should make the
// operator clean up
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.ObjectMeta.Annotations = map[string]string{}
})
expectReconciled(t, sr, "default", "test")
// // synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet
// // didn't create any child resources since this is all faked, so the
// // deletion goes through immediately.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
// // The deletion triggers another reconcile, to finish the cleanup.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
// At the moment we don't revert changes to the user created Service -
// we don't have a reliable way how to tell what it was before and also
// we don't really expect it to be re-used
}
func TestAnnotations(t *testing.T) {
fc := fake.NewFakeClient()
@ -781,8 +887,8 @@ func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"tailscale.com/operator-last-set-hostname": hostname,
"tailscale.com/operator-last-set-ip": "10.20.30.40",
"tailscale.com/operator-last-set-hostname": hostname,
"tailscale.com/operator-last-set-cluster-ip": "10.20.30.40",
},
DeletionGracePeriodSeconds: ptr.To[int64](10),
Labels: map[string]string{"app": "1234-UID"},
@ -825,6 +931,75 @@ func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv
},
}
}
func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityClassName string) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: stsName,
Namespace: "operator-ns",
Labels: map[string]string{
"tailscale.com/managed": "true",
"tailscale.com/parent-resource": "test",
"tailscale.com/parent-resource-ns": "default",
"tailscale.com/parent-resource-type": "svc",
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: ptr.To[int32](1),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "1234-UID"},
},
ServiceName: stsName,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"tailscale.com/operator-last-set-hostname": hostname,
"tailscale.com/operator-last-set-ts-tailnet-target-ip": tailnetTargetIP,
},
DeletionGracePeriodSeconds: ptr.To[int64](10),
Labels: map[string]string{"app": "1234-UID"},
},
Spec: corev1.PodSpec{
ServiceAccountName: "proxies",
PriorityClassName: priorityClassName,
InitContainers: []corev1.Container{
{
Name: "sysctler",
Image: "busybox",
Command: []string{"/bin/sh"},
Args: []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"},
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
},
},
},
Containers: []corev1.Container{
{
Name: "tailscale",
Image: "tailscale/tailscale",
Env: []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "false"},
{Name: "TS_AUTH_ONCE", Value: "true"},
{Name: "TS_KUBE_SECRET", Value: secretName},
{Name: "TS_HOSTNAME", Value: hostname},
{Name: "TS_TAILNET_TARGET_IP", Value: tailnetTargetIP},
},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
},
},
ImagePullPolicy: "Always",
},
},
},
},
},
}
}
func findGenName(t *testing.T, client client.Client, ns, name string) (full, noSuffix string) {
t.Helper()

47
cmd/k8s-operator/sts.go
View File

@ -38,17 +38,19 @@ const (
FinalizerName = "tailscale.com/finalizer"
// Annotations settable by users on services.
AnnotationExpose = "tailscale.com/expose"
AnnotationTags = "tailscale.com/tags"
AnnotationHostname = "tailscale.com/hostname"
AnnotationExpose = "tailscale.com/expose"
AnnotationTags = "tailscale.com/tags"
AnnotationHostname = "tailscale.com/hostname"
AnnotationTailnetTargetIP = "tailscale.com/ts-tailnet-target-ip"
// Annotations settable by users on ingresses.
AnnotationFunnel = "tailscale.com/funnel"
// Annotations set by the operator on pods to trigger restarts when the
// hostname or IP changes.
podAnnotationLastSetIP = "tailscale.com/operator-last-set-ip"
podAnnotationLastSetHostname = "tailscale.com/operator-last-set-hostname"
podAnnotationLastSetClusterIP = "tailscale.com/operator-last-set-cluster-ip"
podAnnotationLastSetHostname = "tailscale.com/operator-last-set-hostname"
podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip"
)
type tailscaleSTSConfig struct {
@ -57,7 +59,11 @@ type tailscaleSTSConfig struct {
ChildResourceLabels map[string]string
ServeConfig *ipn.ServeConfig
TargetIP string
// Tailscale target in cluster we are setting up ingress for
ClusterTargetIP string
// Tailscale IP of a Tailscale service we are setting up egress for
TailnetTargetIP string
Hostname string
Tags []string // if empty, use defaultTags
@ -74,23 +80,23 @@ type tailscaleSTSReconciler struct {
// Provision ensures that the StatefulSet for the given service is running and
// up to date.
func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) error {
func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) {
// Do full reconcile.
hsvc, err := a.reconcileHeadlessService(ctx, logger, sts)
if err != nil {
return fmt.Errorf("failed to reconcile headless service: %w", err)
return nil, fmt.Errorf("failed to reconcile headless service: %w", err)
}
secretName, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
if err != nil {
return fmt.Errorf("failed to create or get API key secret: %w", err)
return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
}
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName)
if err != nil {
return fmt.Errorf("failed to reconcile statefulset: %w", err)
return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
}
return nil
return hsvc, nil
}
// Cleanup removes all resources associated that were created by Provision with
@ -305,11 +311,17 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
Name: "TS_HOSTNAME",
Value: sts.Hostname,
})
if sts.TargetIP != "" {
if sts.ClusterTargetIP != "" {
container.Env = append(container.Env, corev1.EnvVar{
Name: "TS_DEST_IP",
Value: sts.TargetIP,
Value: sts.ClusterTargetIP,
})
} else if sts.TailnetTargetIP != "" {
container.Env = append(container.Env, corev1.EnvVar{
Name: "TS_TAILNET_TARGET_IP",
Value: sts.TailnetTargetIP,
})
} else if sts.ServeConfig != nil {
container.Env = append(container.Env, corev1.EnvVar{
Name: "TS_SERVE_CONFIG",
@ -350,10 +362,13 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
// container when the value changes. We do this by adding an annotation to
// the pod template that contains the last value we set.
ss.Spec.Template.Annotations = map[string]string{
"tailscale.com/operator-last-set-hostname": sts.Hostname,
podAnnotationLastSetHostname: sts.Hostname,
}
if sts.TargetIP != "" {
ss.Spec.Template.Annotations["tailscale.com/operator-last-set-ip"] = sts.TargetIP
if sts.ClusterTargetIP != "" {
ss.Spec.Template.Annotations[podAnnotationLastSetClusterIP] = sts.ClusterTargetIP
}
if sts.TailnetTargetIP != "" {
ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetIP] = sts.TailnetTargetIP
}
ss.Spec.Template.Labels = map[string]string{
"app": sts.ParentResourceUID,

47
cmd/k8s-operator/svc.go
View File

@ -55,8 +55,8 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
} else if err != nil {
return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
}
if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) {
logger.Debugf("service is being deleted or should not be exposed, cleaning up")
if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) && !a.hasTailnetTargetAnnotation(svc) {
logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
}
@ -122,24 +122,34 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
tags = strings.Split(tstr, ",")
}
clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
if err != nil {
return fmt.Errorf("failed to parse cluster IP: %w", err)
}
sts := &tailscaleSTSConfig{
ParentResourceName: svc.Name,
ParentResourceUID: string(svc.UID),
TargetIP: svc.Spec.ClusterIP,
ClusterTargetIP: svc.Spec.ClusterIP,
Hostname: hostname,
Tags: tags,
ChildResourceLabels: crl,
TailnetTargetIP: svc.Annotations[AnnotationTailnetTargetIP],
}
if err := a.ssr.Provision(ctx, logger, sts); err != nil {
var hsvc *corev1.Service
if hsvc, err = a.ssr.Provision(ctx, logger, sts); err != nil {
return fmt.Errorf("failed to provision: %w", err)
}
if a.hasTailnetTargetAnnotation(svc) {
headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc"
if svc.Spec.ExternalName != headlessSvcName || svc.Spec.Type != corev1.ServiceTypeExternalName {
svc.Spec.ExternalName = headlessSvcName
svc.Spec.Selector = nil
svc.Spec.Type = corev1.ServiceTypeExternalName
if err := a.Update(ctx, svc); err != nil {
return fmt.Errorf("failed to update service: %w", err)
}
}
return nil
}
if !a.hasLoadBalancerClass(svc) {
logger.Debugf("service is not a LoadBalancer, so not updating ingress")
return nil
@ -163,6 +173,10 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
ingress := []corev1.LoadBalancerIngress{
{Hostname: tsHost},
}
clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
if err != nil {
return fmt.Errorf("failed to parse cluster IP: %w", err)
}
for _, ip := range tsIPs {
addr, err := netip.ParseAddr(ip)
if err != nil {
@ -186,7 +200,7 @@ func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool {
return false
}
return a.hasLoadBalancerClass(svc) || a.hasAnnotation(svc)
return a.hasLoadBalancerClass(svc) || a.hasExposeAnnotation(svc)
}
func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
@ -196,7 +210,14 @@ func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool {
svc.Spec.LoadBalancerClass == nil && a.isDefaultLoadBalancer)
}
func (a *ServiceReconciler) hasAnnotation(svc *corev1.Service) bool {
return svc != nil &&
svc.Annotations[AnnotationExpose] == "true"
// hasExposeAnnotation reports whether Service has the tailscale.com/expose
// annotation set
func (a *ServiceReconciler) hasExposeAnnotation(svc *corev1.Service) bool {
return svc != nil && svc.Annotations[AnnotationExpose] == "true"
}
// hasTailnetTargetAnnotation reports whether Service has a
// tailscale.com/ts-tailnet-target-ip annotation set
func (a *ServiceReconciler) hasTailnetTargetAnnotation(svc *corev1.Service) bool {
return svc != nil && svc.Annotations[AnnotationTailnetTargetIP] != ""
}