diff --git a/tstest/natlab/vnet/conf.go b/tstest/natlab/vnet/conf.go index 4c8810998..16f9645f9 100644 --- a/tstest/natlab/vnet/conf.go +++ b/tstest/natlab/vnet/conf.go @@ -14,6 +14,7 @@ "github.com/google/gopacket/layers" "github.com/google/gopacket/pcapgo" "tailscale.com/types/logger" + "tailscale.com/util/must" "tailscale.com/util/set" ) @@ -206,7 +207,7 @@ func (s *Server) initFromConfig(c *Config) error { } s.pcapWriter = pw } - for _, conf := range c.networks { + for i, conf := range c.networks { if conf.err != nil { return conf.err } @@ -228,6 +229,14 @@ func (s *Server) initFromConfig(c *Config) error { return fmt.Errorf("two networks have the same WAN IP %v; Anycast not (yet?) supported", conf.wanIP) } s.networkByWAN[conf.wanIP] = n + n.lanInterfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{ + Name: fmt.Sprintf("network%d-lan", i+1), + LinkType: layers.LinkTypeIPv4, + })) + n.wanInterfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{ + Name: fmt.Sprintf("network%d-wan", i+1), + LinkType: layers.LinkTypeIPv4, + })) } for i, conf := range c.nodes { if conf.err != nil { @@ -235,15 +244,12 @@ func (s *Server) initFromConfig(c *Config) error { } n := &node{ mac: conf.mac, - id: i + 1, net: netOfConf[conf.Network()], } - if s.pcapWriter != nil { - s.pcapWriter.w.AddInterface(pcapgo.NgInterface{ - Name: fmt.Sprintf("node%d", n.id), - LinkType: layers.LinkTypeEthernet, - }) - } + n.interfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{ + Name: fmt.Sprintf("node%d", i+1), + LinkType: layers.LinkTypeEthernet, + })) conf.n = n if _, ok := s.nodeByMAC[n.mac]; ok { return fmt.Errorf("two nodes have the same MAC %v", n.mac) diff --git a/tstest/natlab/vnet/pcap.go b/tstest/natlab/vnet/pcap.go index 3647801bc..fa1904667 100644 --- a/tstest/natlab/vnet/pcap.go +++ b/tstest/natlab/vnet/pcap.go @@ -12,6 +12,8 @@ "github.com/google/gopacket/pcapgo" ) +// pcapWriter is a pcapgo.NgWriter that writes to a file. +// It is safe for concurrent use. The nil value is a no-op. type pcapWriter struct { f *os.File @@ -20,6 +22,9 @@ type pcapWriter struct { } func (p *pcapWriter) WritePacket(ci gopacket.CaptureInfo, data []byte) error { + if p == nil { + return nil + } p.mu.Lock() defer p.mu.Unlock() if p.w == nil { @@ -28,7 +33,19 @@ func (p *pcapWriter) WritePacket(ci gopacket.CaptureInfo, data []byte) error { return p.w.WritePacket(ci, data) } +func (p *pcapWriter) AddInterface(i pcapgo.NgInterface) (int, error) { + if p == nil { + return 0, nil + } + p.mu.Lock() + defer p.mu.Unlock() + return p.w.AddInterface(i) +} + func (p *pcapWriter) Close() error { + if p == nil { + return nil + } p.mu.Lock() defer p.mu.Unlock() if p.w != nil { diff --git a/tstest/natlab/vnet/vnet.go b/tstest/natlab/vnet/vnet.go index 4e7e763ef..42a39b9e4 100644 --- a/tstest/natlab/vnet/vnet.go +++ b/tstest/natlab/vnet/vnet.go @@ -404,13 +404,15 @@ type portMapping struct { } type network struct { - s *Server - mac MAC - portmap bool - wanIP netip.Addr - lanIP netip.Prefix // with host bits set (e.g. 192.168.2.1/24) - nodesByIP map[netip.Addr]*node - logf func(format string, args ...any) + s *Server + mac MAC + portmap bool + lanInterfaceID int + wanInterfaceID int + wanIP netip.Addr + lanIP netip.Prefix // with host bits set (e.g. 192.168.2.1/24) + nodesByIP map[netip.Addr]*node + logf func(format string, args ...any) ns *stack.Stack linkEP *channel.Endpoint @@ -445,10 +447,10 @@ func (n *network) MACOfIP(ip netip.Addr) (_ MAC, ok bool) { } type node struct { - mac MAC - id int - net *network - lanIP netip.Addr // must be in net.lanIP prefix + unique in net + mac MAC + interfaceID int + net *network + lanIP netip.Addr // must be in net.lanIP prefix + unique in net } type derpServer struct { @@ -570,9 +572,7 @@ func New(c *Config) (*Server, error) { func (s *Server) Close() { if shutdown := s.shuttingDown.Swap(true); !shutdown { s.shutdownCancel() - if s.pcapWriter != nil { - s.pcapWriter.Close() - } + s.pcapWriter.Close() } s.wg.Wait() } @@ -647,17 +647,12 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) { if err := bw.Flush(); err != nil { log.Printf("Flush: %v", err) } - if s.pcapWriter != nil { - ci := gopacket.CaptureInfo{ - Timestamp: time.Now(), - CaptureLength: len(pkt), - Length: len(pkt), - } - if srcNode != nil { - ci.InterfaceIndex = srcNode.id - } - must.Do(s.pcapWriter.WritePacket(ci, pkt)) - } + must.Do(s.pcapWriter.WritePacket(gopacket.CaptureInfo{ + Timestamp: time.Now(), + CaptureLength: len(pkt), + Length: len(pkt), + InterfaceIndex: srcNode.interfaceID, + }, pkt)) } buf := make([]byte, 16<<10) @@ -717,17 +712,12 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) { continue } } - if s.pcapWriter != nil { - ci := gopacket.CaptureInfo{ - Timestamp: time.Now(), - CaptureLength: len(packetRaw), - Length: len(packetRaw), - } - if srcNode != nil { - ci.InterfaceIndex = srcNode.id - } - must.Do(s.pcapWriter.WritePacket(ci, packetRaw)) - } + must.Do(s.pcapWriter.WritePacket(gopacket.CaptureInfo{ + Timestamp: time.Now(), + CaptureLength: len(packetRaw), + Length: len(packetRaw), + InterfaceIndex: srcNode.interfaceID, + }, packetRaw)) netw.HandleEthernetPacket(ep) } } @@ -825,12 +815,34 @@ func (n *network) HandleEthernetPacket(ep EthernetPacket) { // LAN IP here and wrapped in an ethernet layer and delivered // to the network. func (n *network) HandleUDPPacket(p UDPPacket) { + buf, err := n.serializedUDPPacket(p.Src, p.Dst, p.Payload, nil) + if err != nil { + n.logf("serializing UDP packet: %v", err) + return + } + n.s.pcapWriter.WritePacket(gopacket.CaptureInfo{ + Timestamp: time.Now(), + CaptureLength: len(buf), + Length: len(buf), + InterfaceIndex: n.wanInterfaceID, + }, buf) dst := n.doNATIn(p.Src, p.Dst) if !dst.IsValid() { n.logf("Warning: NAT dropped packet; no mapping for %v=>%v", p.Src, p.Dst) return } p.Dst = dst + buf, err = n.serializedUDPPacket(p.Src, p.Dst, p.Payload, nil) + if err != nil { + n.logf("serializing UDP packet: %v", err) + return + } + n.s.pcapWriter.WritePacket(gopacket.CaptureInfo{ + Timestamp: time.Now(), + CaptureLength: len(buf), + Length: len(buf), + InterfaceIndex: n.lanInterfaceID, + }, buf) n.WriteUDPPacketNoNAT(p) } @@ -853,6 +865,20 @@ func (n *network) WriteUDPPacketNoNAT(p UDPPacket) { DstMAC: node.mac.HWAddr(), EthernetType: layers.EthernetTypeIPv4, } + ethRaw, err := n.serializedUDPPacket(src, dst, p.Payload, eth) + if err != nil { + n.logf("serializing UDP packet: %v", err) + return + } + n.writeEth(ethRaw) +} + +// serializedUDPPacket serializes a UDP packet with the given source and +// destination IP:port pairs, and payload. +// +// If eth is non-nil, it will be used as the Ethernet layer, otherwise the +// Ethernet layer will be omitted from the serialization. +func (n *network) serializedUDPPacket(src, dst netip.AddrPort, payload []byte, eth *layers.Ethernet) ([]byte, error) { ip := &layers.IPv4{ Version: 4, TTL: 64, @@ -868,12 +894,14 @@ func (n *network) WriteUDPPacketNoNAT(p UDPPacket) { buffer := gopacket.NewSerializeBuffer() options := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true} - if err := gopacket.SerializeLayers(buffer, options, eth, ip, udp, gopacket.Payload(p.Payload)); err != nil { - n.logf("serializing UDP: %v", err) - return + layers := []gopacket.SerializableLayer{eth, ip, udp, gopacket.Payload(payload)} + if eth == nil { + layers = layers[1:] } - ethRaw := buffer.Bytes() - n.writeEth(ethRaw) + if err := gopacket.SerializeLayers(buffer, options, layers...); err != nil { + return nil, fmt.Errorf("serializing UDP: %v", err) + } + return buffer.Bytes(), nil } // HandleEthernetIPv4PacketForRouter handles an IPv4 packet that is @@ -931,10 +959,30 @@ func (n *network) HandleEthernetIPv4PacketForRouter(ep EthernetPacket) { if toForward && isUDP { src := netip.AddrPortFrom(srcIP, uint16(udp.SrcPort)) dst := netip.AddrPortFrom(dstIP, uint16(udp.DstPort)) - src0 := src + buf, err := n.serializedUDPPacket(src, dst, udp.Payload, nil) + if err != nil { + n.logf("serializing UDP packet: %v", err) + return + } + n.s.pcapWriter.WritePacket(gopacket.CaptureInfo{ + Timestamp: time.Now(), + CaptureLength: len(buf), + Length: len(buf), + InterfaceIndex: n.lanInterfaceID, + }, buf) + src = n.doNATOut(src, dst) - _ = src0 - //log.Printf("XXX UDP out %v=>%v to %v", src0, src, dst) + buf, err = n.serializedUDPPacket(src, dst, udp.Payload, nil) + if err != nil { + n.logf("serializing UDP packet: %v", err) + return + } + n.s.pcapWriter.WritePacket(gopacket.CaptureInfo{ + Timestamp: time.Now(), + CaptureLength: len(buf), + Length: len(buf), + InterfaceIndex: n.wanInterfaceID, + }, buf) n.s.routeUDPPacket(UDPPacket{ Src: src,