e48cddfbb3
Implements the controller for the new ProxyGroup CRD, designed for running proxies in a high availability configuration. Each proxy gets its own config and state Secret, and its own tailscale node ID. We are currently mounting all of the config secrets into the container, but will stop mounting them and instead read them directly from the kube API once #13578 is implemented. Updates #13406 Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
269 lines
8.3 KiB
Go
269 lines
8.3 KiB
Go
// Copyright (c) Tailscale Inc & AUTHORS
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
//go:build !plan9
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/AlekSi/pointer"
|
|
"github.com/google/go-cmp/cmp"
|
|
"go.uber.org/zap"
|
|
corev1 "k8s.io/api/core/v1"
|
|
discoveryv1 "k8s.io/api/discovery/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
|
"tailscale.com/kube/egressservices"
|
|
"tailscale.com/tstest"
|
|
"tailscale.com/tstime"
|
|
)
|
|
|
|
func TestTailscaleEgressServices(t *testing.T) {
|
|
pg := &tsapi.ProxyGroup{
|
|
TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "foo",
|
|
UID: types.UID("1234-UID"),
|
|
},
|
|
Spec: tsapi.ProxyGroupSpec{
|
|
Replicas: pointer.To[int32](3),
|
|
Type: tsapi.ProxyGroupTypeEgress,
|
|
},
|
|
}
|
|
cm := &corev1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: fmt.Sprintf(egressSvcsCMNameTemplate, "foo"),
|
|
Namespace: "operator-ns",
|
|
},
|
|
}
|
|
fc := fake.NewClientBuilder().
|
|
WithScheme(tsapi.GlobalScheme).
|
|
WithObjects(pg, cm).
|
|
WithStatusSubresource(pg).
|
|
Build()
|
|
zl, err := zap.NewDevelopment()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
clock := tstest.NewClock(tstest.ClockOpts{})
|
|
|
|
esr := &egressSvcsReconciler{
|
|
Client: fc,
|
|
logger: zl.Sugar(),
|
|
clock: clock,
|
|
tsNamespace: "operator-ns",
|
|
}
|
|
tailnetTargetFQDN := "foo.bar.ts.net."
|
|
svc := &corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test",
|
|
Namespace: "default",
|
|
UID: types.UID("1234-UID"),
|
|
Annotations: map[string]string{
|
|
AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
|
|
AnnotationProxyGroup: "foo",
|
|
},
|
|
},
|
|
Spec: corev1.ServiceSpec{
|
|
ExternalName: "placeholder",
|
|
Type: corev1.ServiceTypeExternalName,
|
|
Selector: nil,
|
|
Ports: []corev1.ServicePort{
|
|
{
|
|
Name: "http",
|
|
Protocol: "TCP",
|
|
Port: 80,
|
|
},
|
|
{
|
|
Name: "https",
|
|
Protocol: "TCP",
|
|
Port: 443,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
t.Run("proxy_group_not_ready", func(t *testing.T) {
|
|
mustCreate(t, fc, svc)
|
|
expectReconciled(t, esr, "default", "test")
|
|
// Service should have EgressSvcValid condition set to Unknown.
|
|
svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
|
|
expectEqual(t, fc, svc, nil)
|
|
})
|
|
|
|
t.Run("proxy_group_ready", func(t *testing.T) {
|
|
mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) {
|
|
pg.Status.Conditions = []metav1.Condition{
|
|
condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
|
|
}
|
|
})
|
|
// Quirks of the fake client.
|
|
mustUpdateStatus(t, fc, "default", "test", func(svc *corev1.Service) {
|
|
svc.Status.Conditions = []metav1.Condition{}
|
|
})
|
|
expectReconciled(t, esr, "default", "test")
|
|
// Verify that a ClusterIP Service has been created.
|
|
name := findGenNameForEgressSvcResources(t, fc, svc)
|
|
expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
|
|
clusterSvc := mustGetClusterIPSvc(t, fc, name)
|
|
// Verify that an EndpointSlice has been created.
|
|
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
|
|
// Verify that ConfigMap contains configuration for the new egress service.
|
|
mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
|
|
r := svcConfiguredReason(svc, true, zl.Sugar())
|
|
// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
|
|
// CluterIP Service.
|
|
svc.Status.Conditions = []metav1.Condition{
|
|
condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
|
|
}
|
|
svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
|
|
svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
|
|
expectEqual(t, fc, svc, nil)
|
|
})
|
|
|
|
t.Run("delete_external_name_service", func(t *testing.T) {
|
|
name := findGenNameForEgressSvcResources(t, fc, svc)
|
|
if err := fc.Delete(context.Background(), svc); err != nil {
|
|
t.Fatalf("error deleting ExternalName Service: %v", err)
|
|
}
|
|
expectReconciled(t, esr, "default", "test")
|
|
// Verify that ClusterIP Service and EndpointSlice have been deleted.
|
|
expectMissing[corev1.Service](t, fc, "operator-ns", name)
|
|
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
|
|
// Verify that service config has been deleted from the ConfigMap.
|
|
mustNotHaveConfigForSvc(t, fc, svc, cm)
|
|
})
|
|
}
|
|
|
|
func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition {
|
|
return metav1.Condition{
|
|
Type: string(typ),
|
|
Status: st,
|
|
LastTransitionTime: conditionTime(clock),
|
|
Reason: r,
|
|
Message: msg,
|
|
}
|
|
}
|
|
|
|
func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string {
|
|
t.Helper()
|
|
labels := egressSvcChildResourceLabels(svc)
|
|
s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels)
|
|
if err != nil {
|
|
t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err)
|
|
}
|
|
if s == nil {
|
|
t.Fatalf("no ClusterIP Service found for ExternalName Service %q", svc.Name)
|
|
}
|
|
return s.GetName()
|
|
}
|
|
|
|
func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
|
|
labels := egressSvcChildResourceLabels(extNSvc)
|
|
return &corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: "operator-ns",
|
|
GenerateName: fmt.Sprintf("ts-%s-", extNSvc.Name),
|
|
Labels: labels,
|
|
},
|
|
Spec: corev1.ServiceSpec{
|
|
Type: corev1.ServiceTypeClusterIP,
|
|
Ports: extNSvc.Spec.Ports,
|
|
},
|
|
}
|
|
}
|
|
|
|
func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Service {
|
|
svc := &corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: "operator-ns",
|
|
},
|
|
}
|
|
if err := cl.Get(context.Background(), client.ObjectKeyFromObject(svc), svc); err != nil {
|
|
t.Fatalf("error retrieving Service")
|
|
}
|
|
return svc
|
|
}
|
|
|
|
func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service) *discoveryv1.EndpointSlice {
|
|
labels := egressSvcChildResourceLabels(extNSvc)
|
|
labels[discoveryv1.LabelManagedBy] = "tailscale.com"
|
|
labels[discoveryv1.LabelServiceName] = name
|
|
return &discoveryv1.EndpointSlice{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: fmt.Sprintf("%s-ipv4", name),
|
|
Namespace: "operator-ns",
|
|
Labels: labels,
|
|
},
|
|
Ports: portsForEndpointSlice(clusterIPSvc),
|
|
AddressType: discoveryv1.AddressTypeIPv4,
|
|
}
|
|
}
|
|
|
|
func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort {
|
|
ports := make([]discoveryv1.EndpointPort, 0)
|
|
for _, p := range svc.Spec.Ports {
|
|
ports = append(ports, discoveryv1.EndpointPort{
|
|
Name: &p.Name,
|
|
Protocol: &p.Protocol,
|
|
Port: pointer.ToInt32(p.TargetPort.IntVal),
|
|
})
|
|
}
|
|
return ports
|
|
}
|
|
|
|
func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) {
|
|
t.Helper()
|
|
wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc)
|
|
if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
|
|
t.Fatalf("Error retrieving ConfigMap: %v", err)
|
|
}
|
|
name := tailnetSvcName(extNSvc)
|
|
gotCfg := configFromCM(t, cm, name)
|
|
if gotCfg == nil {
|
|
t.Fatalf("No config found for service %q", name)
|
|
}
|
|
if diff := cmp.Diff(*gotCfg, wantsCfg); diff != "" {
|
|
t.Fatalf("unexpected config for service %q (-got +want):\n%s", name, diff)
|
|
}
|
|
}
|
|
|
|
func mustNotHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc *corev1.Service, cm *corev1.ConfigMap) {
|
|
t.Helper()
|
|
if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
|
|
t.Fatalf("Error retrieving ConfigMap: %v", err)
|
|
}
|
|
name := tailnetSvcName(extNSvc)
|
|
gotCfg := configFromCM(t, cm, name)
|
|
if gotCfg != nil {
|
|
t.Fatalf("Config %#+v for service %q found when it should not be present", gotCfg, name)
|
|
}
|
|
}
|
|
|
|
func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressservices.Config {
|
|
t.Helper()
|
|
cfgBs, ok := cm.BinaryData[egressservices.KeyEgressServices]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
cfgs := &egressservices.Configs{}
|
|
if err := json.Unmarshal(cfgBs, cfgs); err != nil {
|
|
t.Fatalf("error unmarshalling config: %v", err)
|
|
}
|
|
cfg, ok := (*cfgs)[svcName]
|
|
if ok {
|
|
return &cfg
|
|
}
|
|
return nil
|
|
}
|