diff --git a/core/ipc_handler.go b/core/ipc_handler.go index 2e50eb34..2cb8ede5 100644 --- a/core/ipc_handler.go +++ b/core/ipc_handler.go @@ -10,13 +10,15 @@ import ( "slices" "time" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" "github.com/goccy/go-yaml" "google.golang.org/protobuf/encoding/protojson" ) +var ErrIPCStatusHandled = errors.New("nylon IPC response handled") + var pjMarshal = protojson.MarshalOptions{EmitUnpopulated: true} var pjUnmarshal = protojson.UnmarshalOptions{DiscardUnknown: true} @@ -63,7 +65,7 @@ func HandleNylonIPC(n *Nylon, rw *bufio.ReadWriter) error { if err := writeResponse(rw, errResponse(err.Error())); err != nil { return err } - return device.ErrIPCStatusHandled + return ErrIPCStatusHandled } // trace is blocking, so we dont dispatch @@ -75,7 +77,7 @@ func HandleNylonIPC(n *Nylon, rw *bufio.ReadWriter) error { if err := writeResponse(rw, resp); err != nil { return err } - return device.ErrIPCStatusHandled + return ErrIPCStatusHandled } done := make(chan *protocol.IpcResponse, 1) @@ -105,7 +107,7 @@ func HandleNylonIPC(n *Nylon, rw *bufio.ReadWriter) error { if err := writeResponse(rw, resp); err != nil { return err } - return device.ErrIPCStatusHandled + return ErrIPCStatusHandled } func handleStatus(n *Nylon, req *protocol.StatusRequest) *protocol.IpcResponse { @@ -127,8 +129,11 @@ func handleStatus(n *Nylon, req *protocol.StatusRequest) *protocol.IpcResponse { } listenPort := uint32(n.LocalCfg.Port) - if n.Device != nil { - listenPort = uint32(n.Device.ListenPort()) + // The WireGuard transport knows its bound port once it starts. + if n.wireGuard != nil { + if port := n.wireGuard.transport.ListenPort(); port != 0 { + listenPort = uint32(port) + } } return &protocol.IpcResponse{ @@ -188,7 +193,7 @@ func buildSeqnos(n *Nylon) []*protocol.SeqnoEntry { return entries } -func buildNeighbours(n *Nylon, wgStats map[state.NyPublicKey]device.PeerStatus) []*protocol.NeighbourInfo { +func buildNeighbours(n *Nylon, wgStats map[state.NyPublicKey]polyamide.PeerStats) []*protocol.NeighbourInfo { ids := slices.Clone(n.GetPeers(n.LocalCfg.Id)) slices.Sort(ids) neighbours := make([]*protocol.NeighbourInfo, 0, len(ids)) @@ -316,26 +321,28 @@ func advertisementsForNode(n *Nylon, id state.NodeId) []*protocol.Advertisement return ads } -func wireGuardPeerStats(n *Nylon) map[state.NyPublicKey]device.PeerStatus { - stats := make(map[state.NyPublicKey]device.PeerStatus) - if n.Device == nil { +func wireGuardPeerStats(n *Nylon) map[state.NyPublicKey]polyamide.PeerStats { + stats := make(map[state.NyPublicKey]polyamide.PeerStats) + if len(n.Transports) == 0 { return stats } - for _, peer := range n.Device.GetPeers() { - stat := peer.Status() - stats[state.NyPublicKey(stat.PublicKey)] = stat + for _, stat := range n.transportPeers() { + if key, ok := peerPublicKey(stat.Peer); ok { + stats[key] = stat + } } return stats } - -func wireGuardPeerStatsProto(stat device.PeerStatus) *protocol.WireGuardPeerStats { - return &protocol.WireGuardPeerStats{ - LatestHandshakeUnix: stat.LatestHandshakeTime().UnixNano(), - TxBytes: stat.TxBytes, - RxBytes: stat.RxBytes, - PersistentKeepaliveInterval: stat.PersistentKeepaliveInterval, - Endpoint: &stat.Endpoint, +func wireGuardPeerStatsProto(stat polyamide.PeerStats) *protocol.WireGuardPeerStats { + var handshake int64 + if !stat.LastHandshake.IsZero() { + handshake = stat.LastHandshake.UnixNano() + } + endpoint := "" + if stat.PreferredEndpoint != nil { + endpoint = stat.PreferredEndpoint.Address() } + return &protocol.WireGuardPeerStats{LatestHandshakeUnix: handshake, TxBytes: stat.TxBytes, RxBytes: stat.RxBytes, PersistentKeepaliveInterval: uint32(stat.Keepalive / time.Second), Endpoint: &endpoint} } func sourceProto(source state.Source) *protocol.Source { @@ -496,7 +503,7 @@ func handleTrace(n *Nylon, rw *bufio.ReadWriter) error { if err := writeResponse(rw, errResponse("tracing not enabled; restart with --dbg-trace-tc")); err != nil { return err } - return device.ErrIPCStatusHandled + return ErrIPCStatusHandled } // Send initial OK if err := writeResponse(rw, &protocol.IpcResponse{Ok: true}); err != nil { @@ -513,7 +520,7 @@ func handleTrace(n *Nylon, rw *bufio.ReadWriter) error { for { select { case <-ctx.Done(): - return device.ErrIPCStatusHandled + return ErrIPCStatusHandled case msg := <-ch: if str, ok := msg.(string); ok { resp := &protocol.IpcResponse{ @@ -521,7 +528,7 @@ func handleTrace(n *Nylon, rw *bufio.ReadWriter) error { Response: &protocol.IpcResponse_Trace{Trace: &protocol.TraceEvent{Line: str}}, } if err := writeResponse(rw, resp); err != nil { - return device.ErrIPCStatusHandled + return ErrIPCStatusHandled } } } diff --git a/core/nylon.go b/core/nylon.go index 57b74525..ca9f48be 100644 --- a/core/nylon.go +++ b/core/nylon.go @@ -4,21 +4,20 @@ import ( "context" "errors" "log/slog" - "net" "net/netip" "os" "os/signal" "path" "reflect" "runtime" + "slices" "sync" "sync/atomic" "syscall" "time" "github.com/encodeous/nylon/perf" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" - "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/state" "github.com/encodeous/tint" "github.com/jellydator/ttlcache/v3" @@ -58,11 +57,25 @@ type Nylon struct { ConfigPath string // resources - Tun tun.Device - wgUapi net.Listener - Interface string - Device *device.Device - observability *observabilityServer + Tun polyamide.HostDevice + Interface string + // Transports are the registered transports. The first is the default. + Transports []polyamide.Transport + linksForNeighbour func(state.NodeId) []LinkConfig + observability *observabilityServer + peerHandles map[state.NodeId]map[polyamide.Transport]polyamide.Peer + neighbourLinks map[state.NodeId][]neighbourLink + // peerReceived is when each peer last received a packet. It ranks links without health. + peerReceived map[polyamide.Peer]time.Time + tunWorkers sync.WaitGroup + // wireGuard serves legacy UAPI. It is nil for custom runtimes. + wireGuard *legacyWireGuard + resourceCleanup sync.Once + // systemConfigured is set once startup begins configuring the host with + // pre-up commands. Only then does cleanup undo host configuration, so a + // failed start cannot tear down another instance using the same interface. + systemConfigured bool + resourceCleanupErr error // only used for debugging & tests AuxConfig map[string]any @@ -79,7 +92,67 @@ type AppliedSystemState struct { Peers map[state.NodeId]state.NyPublicKey } +// Runtime supplies the transports and host device. Nylon owns them and closes +// them if initialization fails. +type Runtime struct { + // The first transport is the default. It carries configured endpoints. + Transports []polyamide.Transport + // Called on the dispatcher. Nil binds configured endpoints to the default transport. + LinksForNeighbour func(state.NodeId) []LinkConfig + Host polyamide.HostDevice +} + +func (rt Runtime) validate() error { + if len(rt.Transports) == 0 || slices.Contains(rt.Transports, nil) || rt.Host == nil { + return errors.New("runtime requires a host device and at least one transport") + } + for i, transport := range rt.Transports { + if slices.Contains(rt.Transports[:i], transport) { + return errors.New("runtime contains a duplicate transport") + } + } + return nil +} + +func (rt Runtime) close() { + if rt.Host != nil { + _ = rt.Host.Close() + } + for _, transport := range rt.Transports { + if transport != nil { + _ = transport.Close() + } + } +} + +// useRuntime attaches a validated runtime. +func (n *Nylon) useRuntime(rt Runtime) { + n.Transports = slices.Clone(rt.Transports) + n.linksForNeighbour = rt.LinksForNeighbour + n.Tun = rt.Host +} + func NewNylon(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, configPath string, aux map[string]any, opts state.NylonOptions, tunables *state.RouterTunables) (*Nylon, error) { + return newNylon(ccfg, ncfg, logLevel, configPath, aux, opts, tunables, nil) +} + +// NewNylonWithRuntime uses rt instead of creating the platform runtime. +func NewNylonWithRuntime(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, configPath string, aux map[string]any, opts state.NylonOptions, tunables *state.RouterTunables, rt Runtime) (*Nylon, error) { + return newNylon(ccfg, ncfg, logLevel, configPath, aux, opts, tunables, &rt) +} + +func newNylon(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, configPath string, aux map[string]any, opts state.NylonOptions, tunables *state.RouterTunables, supplied *Runtime) (*Nylon, error) { + attached := false + defer func() { + if !attached && supplied != nil { + supplied.close() + } + }() + if supplied != nil { + if err := supplied.validate(); err != nil { + return nil, err + } + } ctx, cancel := context.WithCancelCause(context.Background()) dispatch := make(chan func() error, 128) @@ -155,12 +228,19 @@ func NewNylon(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, c AuxConfig: aux, DNSResolver: dnsResolver, EndpointResolver: state.NewEndpointResolver(dnsResolver), + peerHandles: make(map[state.NodeId]map[polyamide.Transport]polyamide.Peer), + } + if supplied != nil { + n.useRuntime(*supplied) } + attached = true n.Log.Info("init modules") err = n.Init() if err != nil { + n.Cancel(err) + _ = n.Cleanup() return nil, err } return n, nil @@ -196,8 +276,7 @@ func (n *Nylon) Init() error { return nylonGc(n) }, n.GcDelay) - // wireguard configuration - err = n.initWireGuard() + err = n.initTransport() if err != nil { return err } @@ -327,16 +406,26 @@ endLoop: } func (n *Nylon) Cleanup() error { + n.resourceCleanup.Do(func() { n.resourceCleanupErr = n.cleanup() }) + return n.resourceCleanupErr +} + +func (n *Nylon) cleanup() error { + n.Cancel(context.Canceled) if n.observability != nil { n.observability.close() } - n.PingBuf.Stop() + if n.PingBuf != nil { + n.PingBuf.Stop() + } for _, health := range n.prefixHealth { health.monitor.Stop() } + err := n.cleanupTransport() n.CleanupRouter() - n.Trace.Cleanup() - - return n.cleanupWireGuard() + if n.Trace != nil && n.Trace.Broadcaster != nil { + _ = n.Trace.Cleanup() + } + return err } diff --git a/core/nylon_apply.go b/core/nylon_apply.go index c9b0e55e..5ea51dc4 100644 --- a/core/nylon_apply.go +++ b/core/nylon_apply.go @@ -46,10 +46,10 @@ func (n *Nylon) ApplyCentralConfig(cfg *state.CentralCfg) (ApplyResult, error) { } func (n *Nylon) SyncApplicationState() error { - if n.Device == nil { + if len(n.Transports) == 0 { return nil } - if err := n.SyncWireGuard(); err != nil { + if err := n.SyncTransport(); err != nil { return err } ComputeRoutes(n.RouterState, n) @@ -105,7 +105,7 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error { Eps: make([]state.Endpoint, 0, len(cfg.Endpoints)), } for _, ep := range cfg.Endpoints { - stNeigh.Eps = append(stNeigh.Eps, state.NewEndpoint(ep, false, nil, &n.RouterTunables)) + stNeigh.Eps = append(stNeigh.Eps, state.NewEndpoint(ep, false, &n.RouterTunables)) } neighs = append(neighs, stNeigh) } @@ -165,7 +165,7 @@ func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []string, t *s if _, ok := seen[address]; ok { continue } - eps = append(eps, state.NewEndpoint(address, false, nil, t)) + eps = append(eps, state.NewEndpoint(address, false, t)) } neigh.Eps = eps } diff --git a/core/nylon_apply_test.go b/core/nylon_apply_test.go index 4ffa781b..e34166a2 100644 --- a/core/nylon_apply_test.go +++ b/core/nylon_apply_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/state" "github.com/gaissmai/bart" "github.com/stretchr/testify/assert" @@ -107,43 +107,21 @@ func TestReconcileAdvertisedPrefixesReusesUnchangedMonitor(t *testing.T) { assert.Equal(t, state.INF, n.RouterState.Advertised[prefix].MetricFn()) } -func TestRebindForwardingPeersUpdatesUnchangedNextHop(t *testing.T) { +func TestPublishLinksPreservesOldSnapshot(t *testing.T) { prefix := netip.MustParsePrefix("10.0.0.0/24") - nextHop := state.NodeId("next") - oldPeer := new(device.Peer) - newPeer := new(device.Peer) + oldPeer := new(testTransportPeer) + newPeer := &testTransportPeer{id: "new"} forward := new(bart.Table[RouteTableEntry]) - forward.Insert(prefix, RouteTableEntry{Nh: nextHop, Peer: oldPeer}) - - rebound, changed := rebindRouteTablePeers(forward, map[state.NodeId]*device.Peer{ - nextHop: newPeer, - }) - - assert.True(t, changed) - forwardEntry, ok := rebound.Lookup(prefix.Addr()) - if assert.True(t, ok) { - assert.Same(t, newPeer, forwardEntry.Peer) - assert.Equal(t, nextHop, forwardEntry.Nh) - } - oldEntry, ok := forward.Lookup(prefix.Addr()) - if assert.True(t, ok) { - assert.Same(t, oldPeer, oldEntry.Peer, "published forwarding snapshots must remain immutable") - } -} - -func TestRebindForwardingPeersReusesUnchangedTable(t *testing.T) { - prefix := netip.MustParsePrefix("10.0.0.0/24") - nextHop := state.NodeId("next") - peer := new(device.Peer) - forward := new(bart.Table[RouteTableEntry]) - forward.Insert(prefix, RouteTableEntry{Nh: nextHop, Peer: peer}) - - rebound, changed := rebindRouteTablePeers(forward, map[state.NodeId]*device.Peer{ - nextHop: peer, - }) - - assert.False(t, changed) - assert.Same(t, forward, rebound) + forward.Insert(prefix, RouteTableEntry{Nh: "next"}) + n := new(Nylon) + old := &ForwardingTables{Forward: forward, Exit: new(bart.Table[RouteTableEntry]), Links: map[state.NodeId][]Link{"next": {{Peer: oldPeer}}}} + n.router.Tables.Store(old) + n.neighbourLinks = map[state.NodeId][]neighbourLink{"next": {{Link: Link{Peer: newPeer}}}} + n.publishLinks() + next := n.router.Tables.Load() + assert.Same(t, forward, next.Forward) + assert.Same(t, newPeer, next.Links["next"][0].Peer) + assert.Same(t, oldPeer, old.Links["next"][0].Peer) } func TestHandleNylonPacketDropsUnknownRetiredPeer(t *testing.T) { @@ -152,7 +130,7 @@ func TestHandleNylonPacketDropsUnknownRetiredPeer(t *testing.T) { n.PeerMap.Store(&peerMap) assert.NotPanics(t, func() { - n.handleNylonPacket(nil, nil, new(device.Peer)) + n.handleNylonPacket(nil, nil, new(testTransportPeer)) }) } @@ -191,3 +169,10 @@ func testCentralConfig(id state.NodeId, prefixes ...state.PrefixHealthWrapper) s }, } } + +type testTransportPeer struct{ id string } + +func (p *testTransportPeer) ID() string { return p.id } +func (p *testTransportPeer) PublicKey() polyamide.PublicKey { return polyamide.PublicKey{} } + +func (p *testTransportPeer) Transport() polyamide.Transport { return nil } diff --git a/core/nylon_cleanup_test.go b/core/nylon_cleanup_test.go new file mode 100644 index 00000000..358a5da1 --- /dev/null +++ b/core/nylon_cleanup_test.go @@ -0,0 +1,31 @@ +package core + +import ( + "io" + "log/slog" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCleanupRunsDownCommandsOnlyAfterStartupConfiguredHost(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses touch") + } + dir := t.TempDir() + preDown, postDown := filepath.Join(dir, "pre-down"), filepath.Join(dir, "post-down") + n := &Nylon{Log: slog.New(slog.NewTextHandler(io.Discard, nil))} + n.PreDown = []string{"touch " + preDown} + n.PostDown = []string{"touch " + postDown} + // A failed start, such as an interface already owned by another instance, + // must not run down commands that would tear down that instance. + require.NoError(t, n.cleanupTransport()) + require.NoFileExists(t, preDown) + require.NoFileExists(t, postDown) + n.systemConfigured = true + require.NoError(t, n.cleanupTransport()) + require.FileExists(t, preDown) + require.FileExists(t, postDown) +} diff --git a/core/nylon_endpoints.go b/core/nylon_endpoints.go index 6b5065a0..73732093 100644 --- a/core/nylon_endpoints.go +++ b/core/nylon_endpoints.go @@ -7,17 +7,17 @@ import ( "slices" "time" - "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" "github.com/jellydator/ttlcache/v3" ) type EpPing struct { - TimeSent time.Time - Peer state.NodeId - Complete func(protocol.EndpointProbeStatus, time.Duration) + TimeSent time.Time + Peer state.NodeId + Transport polyamide.Transport + Complete func(protocol.EndpointProbeStatus, time.Duration) } func (n *Nylon) sendEndpointProbes(peer state.NodeId, timeout time.Duration) ([]Future[*protocol.EndpointProbeResult], error) { @@ -62,15 +62,25 @@ func (n *Nylon) sendEndpointProbe(node state.NodeId, ep *state.NylonEndpoint, ti return resultFuture, err } - peer := n.Device.LookupPeer(device.NoisePublicKey(n.GetNode(node).PubKey)) - if peer == nil { - return fail(protocol.EndpointProbeStatus_ENDPOINT_PROBE_SEND_ERROR, fmt.Errorf("wireguard peer %q is not configured", node)) + index := slices.IndexFunc(n.neighbourLinks[node], func(link neighbourLink) bool { return link.health == ep }) + if index == -1 { + return fail(protocol.EndpointProbeStatus_ENDPOINT_PROBE_SEND_ERROR, fmt.Errorf("endpoint for neighbour %q is not configured", node)) } - nep, err := ep.GetWgEndpoint(n.Device, n.EndpointResolver) + // Refresh the native endpoint when DNS changes the destination. + link := n.neighbourLinks[node][index].Link + current, err := n.EndpointResolver.Get(ep.Address) if err != nil { return fail(protocol.EndpointProbeStatus_ENDPOINT_PROBE_RESOLVE_ERROR, err) } - resolved = nep.DstIPPort().String() + if link.Endpoint == nil || link.Endpoint.Address() != current.String() { + endpoint, err := link.Peer.Transport().PrepareEndpoint(n.Context, current.String()) + if err != nil { + return fail(protocol.EndpointProbeStatus_ENDPOINT_PROBE_RESOLVE_ERROR, err) + } + link.Endpoint = endpoint + n.neighbourLinks[node][index].Endpoint = endpoint + } + resolved = link.Endpoint.Address() token := rand.Uint64() sentAt := time.Now() ping := &protocol.Ny{ @@ -90,8 +100,9 @@ func (n *Nylon) sendEndpointProbe(node state.NodeId, ep *state.NylonEndpoint, ti } n.PingBuf.Set(token, EpPing{ - TimeSent: sentAt, - Peer: node, + TimeSent: sentAt, + Peer: node, + Transport: link.Peer.Transport(), Complete: func(status protocol.EndpointProbeStatus, latency time.Duration) { if timeoutTimer != nil { timeoutTimer.Stop() @@ -101,7 +112,7 @@ func (n *Nylon) sendEndpointProbe(node state.NodeId, ep *state.NylonEndpoint, ti }, ttlcache.DefaultTTL) go func() { - err := n.SendNylon(ping, nep, peer) + err := n.SendNylon(ping, link.Endpoint, link.Peer) if err != nil { if timeoutTimer != nil { timeoutTimer.Stop() @@ -114,7 +125,7 @@ func (n *Nylon) sendEndpointProbe(node state.NodeId, ep *state.NylonEndpoint, ti return resultFuture, nil } -func handleProbe(n *Nylon, pkt *protocol.Ny_Probe, endpoint conn.Endpoint, peer *device.Peer, node state.NodeId) { +func handleProbe(n *Nylon, pkt *protocol.Ny_Probe, endpoint polyamide.Endpoint, peer polyamide.Peer, node state.NodeId) { if pkt.ResponseToken == nil { // ping // build pong response @@ -132,100 +143,100 @@ func handleProbe(n *Nylon, pkt *protocol.Ny_Probe, endpoint conn.Endpoint, peer } n.Dispatch(func() error { - handleProbePing(n, node, endpoint) + handleProbePing(n, node, endpoint, peer) return nil }) } else { // pong n.Dispatch(func() error { - handleProbePong(n, node, pkt.Token, endpoint) + handleProbePong(n, node, pkt.Token, endpoint, peer) return nil }) } } -func handleProbePing(n *Nylon, node state.NodeId, wgEndpoint conn.Endpoint) { - if node == n.LocalCfg.Id { - return +func (n *Nylon) receivedLink(node state.NodeId, endpoint polyamide.Endpoint, peer polyamide.Peer) *state.NylonEndpoint { + if endpoint == nil || peer == nil || n.transportPeer(node, peer.Transport()) != peer { + return nil } - // check if link exists - for _, neigh := range n.RouterState.Neighbours { - for _, dep := range neigh.Eps { - dep := dep.AsNylonEndpoint() - ap, err := n.EndpointResolver.Get(dep.Address) - if err == nil && ap == wgEndpoint.DstIPPort() && neigh.Id == node { - // we have a link + for i, link := range n.neighbourLinks[node] { + if link.health != nil && link.Peer == peer && n.linkAddress(link) == endpoint.Address() { + n.neighbourLinks[node][i].Endpoint = endpoint + return link.health + } + } + neigh := n.RouterState.GetNeighbour(node) + if neigh == nil { + return nil + } + health := state.NewEndpoint(endpoint.Address(), true, &n.RouterTunables) + neigh.Eps = append(neigh.Eps, health) + if n.neighbourLinks == nil { + n.neighbourLinks = make(map[state.NodeId][]neighbourLink) + } + n.neighbourLinks[node] = append(n.neighbourLinks[node], neighbourLink{Link: Link{Peer: peer, Endpoint: endpoint}, health: health}) + return health +} - // refresh wireguard ep - dep.WgEndpoint = wgEndpoint +func handleProbePing(n *Nylon, node state.NodeId, endpoint polyamide.Endpoint, peer polyamide.Peer) { + n.renewLink(node, endpoint, peer) +} - wasInactive := !dep.IsActive() - dep.Renew() - if wasInactive { - ComputeRoutes(n.RouterState, n) - n.UpdateNeighbour(node) - } +// endpointLearned handles a packet from peer at an endpoint its transport was not +// configured with, such as after the peer roams. +func (n *Nylon) endpointLearned(node state.NodeId, endpoint polyamide.Endpoint, peer polyamide.Peer) { + if n.transportPeer(node, peer.Transport()) != peer { + return // the handle was retired + } + if n.IsClient(node) { + n.passiveClientRoamed(peer) + return + } + n.renewLink(node, endpoint, peer) +} - if n.DBG_log_probe { - n.Log.Debug("probe from", "addr", ap.String()) - } - return - } - } +// renewLink records traffic from a neighbour at endpoint, learning it as a link if needed. +func (n *Nylon) renewLink(node state.NodeId, endpoint polyamide.Endpoint, peer polyamide.Peer) { + if node == n.LocalCfg.Id { + return } - // create a new link if we dont have a link - for _, neigh := range n.RouterState.Neighbours { - if neigh.Id == node { - newEp := state.NewEndpoint(wgEndpoint.DstIPPort().String(), true, wgEndpoint, &n.RouterTunables) - newEp.Renew() - neigh.Eps = append(neigh.Eps, newEp) - // push route update to improve convergence time - ComputeRoutes(n.RouterState, n) - n.UpdateNeighbour(node) - return - } + health := n.receivedLink(node, endpoint, peer) + if health == nil { + return + } + wasInactive := !health.IsActive() + health.Renew() + if wasInactive { + ComputeRoutes(n.RouterState, n) + n.UpdateNeighbour(node) } + n.publishLinks() } -func handleProbePong(n *Nylon, node state.NodeId, token uint64, ep conn.Endpoint) { +func handleProbePong(n *Nylon, node state.NodeId, token uint64, ep polyamide.Endpoint, peer polyamide.Peer) { linkHealth, ok := n.PingBuf.GetAndDelete(token) if !ok { return } health := linkHealth.Value() - if health.Peer != "" && health.Peer != node { + if health.Peer != node || health.Transport != peer.Transport() { n.Log.Warn("probe came back from wrong peer", "expected", health.Peer, "actual", node) return } - receivedAt := time.Now() - latency := receivedAt.Sub(health.TimeSent) + latency := time.Since(health.TimeSent) health.Complete(protocol.EndpointProbeStatus_ENDPOINT_PROBE_REPLIED, latency) - - // check if link exists - for _, neigh := range n.RouterState.Neighbours { - for _, dep := range neigh.Eps { - dpLink := dep.AsNylonEndpoint() - ap, err := n.EndpointResolver.Get(dpLink.Address) - if err == nil && ap == ep.DstIPPort() && neigh.Id == node { - // we have a link - if n.DBG_log_probe { - n.Log.Debug("probe back", "peer", node, "ping", latency) - } - dpLink.Renew() - dpLink.UpdatePing(latency) - - // update wireguard endpoint - dpLink.WgEndpoint = ep - - ComputeRoutes(n.RouterState, n) - return - } - } + link := n.receivedLink(node, ep, peer) + if link == nil { + return } - n.Log.Warn("probe came back and couldn't find link", "from", ep.DstToString(), "node", node) + link.Renew() + link.UpdatePing(latency) + ComputeRoutes(n.RouterState, n) + n.publishLinks() } func (n *Nylon) probeLinks(active bool) error { + defer n.publishLinks() // probe links for _, neigh := range n.RouterState.Neighbours { for _, ep := range neigh.Eps { @@ -241,33 +252,16 @@ func (n *Nylon) probeLinks(active bool) error { } func (n *Nylon) probeNew() error { - // probe for new dp links - for _, peer := range n.GetPeers(n.LocalCfg.Id) { - if !n.IsRouter(peer) { - continue - } - neigh := n.RouterState.GetNeighbour(peer) - if neigh == nil { - continue - } - cfg := n.GetRouter(peer) - for _, address := range cfg.Endpoints { - idx := slices.IndexFunc(neigh.Eps, func(link state.Endpoint) bool { - return !link.IsRemote() && link.AsNylonEndpoint().Address == address - }) - if idx == -1 { - // add the link to the neighbour - dpl := state.NewEndpoint(address, false, nil, &n.RouterTunables) - neigh.Eps = append(neigh.Eps, dpl) - idx = len(neigh.Eps) - 1 - } - dpl := neigh.Eps[idx].AsNylonEndpoint() - if _, err := n.EndpointResolver.Get(dpl.Address); err != nil { + // probe configured links for new dp links + for id, links := range n.neighbourLinks { + for _, link := range links { + if link.health == nil || link.health.IsRemote() { continue } - if err := n.Probe(peer, dpl); err != nil { - //n.Log.Debug("discovery probe failed", "err", err.Error()) + if _, err := n.EndpointResolver.Get(link.health.Address); err != nil { + continue } + _ = n.Probe(id, link.health) } } return nil diff --git a/core/nylon_gc.go b/core/nylon_gc.go index 21ac4e61..bc080721 100644 --- a/core/nylon_gc.go +++ b/core/nylon_gc.go @@ -31,6 +31,8 @@ func nylonGc(n *Nylon) error { } neigh.Eps = neigh.Eps[:count] } + n.pruneLinks() + n.publishLinks() err := n.GcRouter() if err != nil { diff --git a/core/nylon_links.go b/core/nylon_links.go new file mode 100644 index 00000000..b32f7074 --- /dev/null +++ b/core/nylon_links.go @@ -0,0 +1,291 @@ +package core + +import ( + "cmp" + "errors" + "fmt" + "slices" + + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/state" +) + +type Link struct { + Peer polyamide.Peer + Endpoint polyamide.Endpoint +} + +type LinkConfig struct { + Transport polyamide.Transport + // Empty allows the transport to learn the peer's endpoint. + Address string +} + +type neighbourLink struct { + Link + health *state.NylonEndpoint +} + +// rankLinks orders links by metric. Links without health rank last, and among +// them, the link whose peer received a packet most recently comes first. +func (n *Nylon) rankLinks(links []neighbourLink) []neighbourLink { + metric := func(link neighbourLink) uint32 { + if link.health == nil { + return state.INF + } + return link.health.Metric() + } + ranked := slices.Clone(links) + slices.SortStableFunc(ranked, func(a, b neighbourLink) int { + return cmp.Or(cmp.Compare(metric(a), metric(b)), n.peerReceived[b.Peer].Compare(n.peerReceived[a.Peer])) + }) + return ranked +} + +// forwardingLinks keeps passive links and links with a finite metric. +func (n *Nylon) forwardingLinks() map[state.NodeId][]Link { + links := make(map[state.NodeId][]Link, len(n.neighbourLinks)) + for id, candidates := range n.neighbourLinks { + for _, link := range n.rankLinks(candidates) { + if link.health == nil || link.health.Metric() != state.INF { + links[id] = append(links[id], link.Link) + } + } + } + return links +} + +func (n *Nylon) bestLink(id state.NodeId) (Link, bool) { + tables := n.router.Tables.Load() + if tables == nil || len(tables.Links[id]) == 0 { + return Link{}, false + } + return tables.Links[id][0], true +} + +// SyncTransport binds every peer to its configured links. A node that fails to +// sync keeps its previous bindings, and the other nodes are still synced. +func (n *Nylon) SyncTransport() error { + if len(n.Transports) == 0 { + return nil + } + ids := slices.Sorted(slices.Values(n.GetPeers(n.LocalCfg.Id))) + result := n.releaseMovedKeys(ids) + desired := make(map[state.NodeId]state.NyPublicKey) + handles := make(map[state.NodeId]map[polyamide.Transport]polyamide.Peer) + links := make(map[state.NodeId][]neighbourLink) + for _, id := range ids { + peers, nodeLinks, err := n.syncNode(id) + if err != nil { + result = errors.Join(result, fmt.Errorf("sync neighbour %s: %w", id, err)) + peers, nodeLinks = n.peerHandles[id], n.neighbourLinks[id] + if key, ok := n.AppliedSystem.Peers[id]; ok { + desired[id] = key + } + } else { + desired[id] = n.GetNode(id).PubKey + } + handles[id], links[id] = peers, nodeLinks + } + // Publish the new bindings before retiring old peer handles. + n.peerHandles, n.neighbourLinks = handles, links + for _, neigh := range n.RouterState.Neighbours { + neigh.Eps = nil + for _, link := range links[neigh.Id] { + if link.health != nil { + neigh.Eps = append(neigh.Eps, link.health) + } + } + } + n.publishLinks() + for _, stat := range n.transportPeers() { + if handles[state.NodeId(stat.Peer.ID())][stat.Peer.Transport()] != stat.Peer { + result = errors.Join(result, stat.Peer.Transport().RemovePeer(n.Context, stat.Peer)) + } + } + for peer := range n.peerReceived { + if handles[state.NodeId(peer.ID())][peer.Transport()] != peer { + delete(n.peerReceived, peer) + } + } + n.AppliedSystem.Peers = desired + return result +} + +// releaseMovedKeys removes peers whose public key now belongs to a different node, +// so the new node can bind it. Keys claimed by several nodes are left alone. +func (n *Nylon) releaseMovedKeys(ids []state.NodeId) error { + owners := make(map[state.NyPublicKey][]state.NodeId) + for _, id := range ids { + key := n.GetNode(id).PubKey + owners[key] = append(owners[key], id) + } + var result error + for _, stat := range n.transportPeers() { + owner := owners[state.NyPublicKey(stat.Peer.PublicKey())] + if len(owner) == 1 && owner[0] != state.NodeId(stat.Peer.ID()) { + result = errors.Join(result, stat.Peer.Transport().RemovePeer(n.Context, stat.Peer)) + } + } + return result +} + +// syncNode prepares a node's peers on each configured transport and gives each +// transport the node's endpoints, best first. +func (n *Nylon) syncNode(id state.NodeId) (map[polyamide.Transport]polyamide.Peer, []neighbourLink, error) { + cfg := n.GetNode(id) + peers := make(map[polyamide.Transport]polyamide.Peer) + var links []neighbourLink + for _, config := range n.configuredLinks(id) { + transport := config.Transport + if transport == nil || !slices.Contains(n.Transports, transport) { + return nil, nil, errors.New("unregistered transport") + } + peer := peers[transport] + if peer == nil { + var err error + peer, err = transport.PreparePeer(n.Context, polyamide.PeerConfig{ID: string(id), PublicKey: polyamide.PublicKey(cfg.PubKey), Passive: n.IsClient(id)}) + if err != nil { + return nil, nil, err + } + peers[transport] = peer + } + // Unresolved addresses are prepared when they are probed. + var endpoint polyamide.Endpoint + if config.Address != "" { + if address, err := n.EndpointResolver.Get(config.Address); err == nil { + // Reuse native state until DNS changes the destination. + for _, old := range n.neighbourLinks[id] { + if old.health == config.health && old.Peer.Transport() == transport && old.Endpoint != nil && old.Endpoint.Address() == address.String() { + endpoint = old.Endpoint + break + } + } + if endpoint == nil { + endpoint, err = transport.PrepareEndpoint(n.Context, address.String()) + if err != nil { + return nil, nil, err + } + } + } + } + links = append(links, neighbourLink{Link: Link{Peer: peer, Endpoint: endpoint}, health: config.health}) + } + // The first endpoint is the transport's default for this peer. + endpoints := make(map[polyamide.Transport][]polyamide.Endpoint) + for _, link := range n.rankLinks(links) { + if link.Endpoint != nil { + endpoints[link.Peer.Transport()] = append(endpoints[link.Peer.Transport()], link.Endpoint) + } + } + for transport, peer := range peers { + if len(endpoints[transport]) == 0 && n.IsClient(id) { + continue + } + if err := transport.SetEndpoints(n.Context, peer, endpoints[transport]); err != nil { + return nil, nil, err + } + } + return peers, links, nil +} + +type configuredLink struct { + LinkConfig + health *state.NylonEndpoint +} + +// defaultLinks binds the configured endpoints of a neighbour to the default transport. +func (n *Nylon) defaultLinks(id state.NodeId) []LinkConfig { + transport := n.defaultTransport() + if transport == nil { + return nil + } + var configs []LinkConfig + if n.IsRouter(id) { + for _, address := range n.GetRouter(id).Endpoints { + configs = append(configs, LinkConfig{Transport: transport, Address: address}) + } + } + if len(configs) == 0 { + configs = append(configs, LinkConfig{Transport: transport}) + } + return configs +} + +func (n *Nylon) configuredLinks(id state.NodeId) []configuredLink { + configs := n.defaultLinks(id) + if n.linksForNeighbour != nil { + configs = n.linksForNeighbour(id) + } + neigh := n.RouterState.GetNeighbour(id) + claimed := make(map[*state.NylonEndpoint]bool) + // boundElsewhere reports whether a health entry already belongs to another transport. + boundElsewhere := func(health *state.NylonEndpoint, transport polyamide.Transport) bool { + return slices.ContainsFunc(n.neighbourLinks[id], func(old neighbourLink) bool { + return old.health == health && old.Peer.Transport() != transport + }) + } + var links []configuredLink + for _, cfg := range configs { + var health *state.NylonEndpoint + // Only neighbours are probed, so only they get link health. + if cfg.Address != "" && neigh != nil { + for _, old := range n.neighbourLinks[id] { + if old.health != nil && !claimed[old.health] && !old.health.IsRemote() && old.Peer.Transport() == cfg.Transport && old.health.Address == cfg.Address { + health = old.health + break + } + } + if health == nil { + // Adopt entries created by router-state reconciliation. + for _, ep := range neigh.Eps { + ep := ep.AsNylonEndpoint() + if !claimed[ep] && !ep.IsRemote() && ep.Address == cfg.Address && !boundElsewhere(ep, cfg.Transport) { + health = ep + break + } + } + } + if health == nil { + health = state.NewEndpoint(cfg.Address, false, &n.RouterTunables) + } + claimed[health] = true + } + links = append(links, configuredLink{LinkConfig: cfg, health: health}) + } + // Keep learned endpoints for transports that still have a configured binding. + for _, old := range n.neighbourLinks[id] { + if old.health == nil || claimed[old.health] || !old.health.IsRemote() || !old.health.IsAlive() { + continue + } + transport := old.Peer.Transport() + if !slices.ContainsFunc(configs, func(c LinkConfig) bool { return c.Transport == transport }) { + continue + } + claimed[old.health] = true + links = append(links, configuredLink{LinkConfig: LinkConfig{Transport: transport, Address: old.health.Address}, health: old.health}) + } + return links +} + +// pruneLinks drops links whose health was garbage collected from router state. +func (n *Nylon) pruneLinks() { + for _, neigh := range n.RouterState.Neighbours { + n.neighbourLinks[neigh.Id] = slices.DeleteFunc(n.neighbourLinks[neigh.Id], func(link neighbourLink) bool { + return link.health != nil && !slices.ContainsFunc(neigh.Eps, func(ep state.Endpoint) bool { return ep.AsNylonEndpoint() == link.health }) + }) + } +} + +// linkAddress is the resolved destination of a link, or "" when it is unknown. +func (n *Nylon) linkAddress(link neighbourLink) string { + if link.Endpoint != nil { + return link.Endpoint.Address() + } + if link.health != nil { + if address, err := n.EndpointResolver.Get(link.health.Address); err == nil { + return address.String() + } + } + return "" +} diff --git a/core/nylon_passive.go b/core/nylon_passive.go index 7e630298..cec80859 100644 --- a/core/nylon_passive.go +++ b/core/nylon_passive.go @@ -1,8 +1,11 @@ package core import ( + "maps" + "slices" "time" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/state" ) @@ -14,42 +17,74 @@ func (n *Nylon) initPassiveClient() error { } func scanPassivePeers(n *Nylon) error { - for _, peer := range n.Device.GetPeers() { - nid := n.FindNodeBy(state.NyPublicKey(peer.GetPublicKey())) - - if nid != nil { - // check if we are the only node that is advertising this passive client, if so, we can apply the following optimization - // As we are the only node advertising the client, we can permanently hold the route (with a very high metric), and not expire it - // This enables our passive client to be reachable even if it does not send any traffic for a long time (e.g. mobile device going to sleep) - // If this device switches to another nylon node, that node will start advertising the client, and we will stop holding the route - - hasOtherAdvertisers := false - ncfg := n.GetNode(*nid) - for _, prefix := range ncfg.Prefixes { - for _, neigh := range n.RouterState.Neighbours { - for _, route := range neigh.Routes { - if route.Prefix == prefix.GetPrefix() && route.NodeId != n.LocalCfg.Id && route.FD.Metric != state.INF { - hasOtherAdvertisers = true - goto foundAdvertiser - } + // A node can have a peer on each transport. Use the most recent packet from any of them. + lastRX := make(map[state.NodeId]time.Time) + for _, stat := range n.transportPeers() { + n.setPeerReceived(stat.Peer, stat.LastReceived) + key, valid := peerPublicKey(stat.Peer) + if !valid { + continue + } + nid := n.FindNodeBy(key) + if nid == nil { + continue + } + if last, seen := lastRX[*nid]; !seen || stat.LastReceived.After(last) { + lastRX[*nid] = stat.LastReceived + } + } + + n.publishLinks() + + for _, nid := range slices.Sorted(maps.Keys(lastRX)) { + if !n.IsClient(nid) { + continue + } + // check if we are the only node that is advertising this passive client, if so, we can apply the following optimization + // As we are the only node advertising the client, we can permanently hold the route (with a very high metric), and not expire it + // This enables our passive client to be reachable even if it does not send any traffic for a long time (e.g. mobile device going to sleep) + // If this device switches to another nylon node, that node will start advertising the client, and we will stop holding the route + + hasOtherAdvertisers := false + ncfg := n.GetNode(nid) + for _, prefix := range ncfg.Prefixes { + for _, neigh := range n.RouterState.Neighbours { + for _, route := range neigh.Routes { + if route.Prefix == prefix.GetPrefix() && route.NodeId != n.LocalCfg.Id && route.FD.Metric != state.INF { + hasOtherAdvertisers = true + goto foundAdvertiser } } } - foundAdvertiser: + } + foundAdvertiser: - // TODO: we could make this expire after a longer period of time, like 24h. However, this would require our passive client to wait for the full route propagation time after 24 hours. (Might cause unexpected interruptions) + // TODO: we could make this expire after a longer period of time, like 24h. However, this would require our passive client to wait for the full route propagation time after 24 hours. (Might cause unexpected interruptions) - recentlyUpdated := time.Since(peer.LastReceivedPacket()) < n.ClientDeadThreshold - if n.IsClient(*nid) { - // we have a passive client - for _, newPrefix := range ncfg.Prefixes { - recentlyAdvertised := n.hasRecentlyAdvertised(newPrefix.GetPrefix()) - if recentlyUpdated || !hasOtherAdvertisers && recentlyAdvertised { - n.updatePassiveClient(newPrefix, *nid, !recentlyUpdated) - } - } + recentlyUpdated := time.Since(lastRX[nid]) < n.ClientDeadThreshold + for _, newPrefix := range ncfg.Prefixes { + recentlyAdvertised := n.hasRecentlyAdvertised(newPrefix.GetPrefix()) + if recentlyUpdated || !hasOtherAdvertisers && recentlyAdvertised { + n.updatePassiveClient(newPrefix, nid, !recentlyUpdated) } } } return nil } + +// passiveClientRoamed forwards to a client through the transport it now uses. +// Clients have no link health, so their links are ranked by recent traffic. +func (n *Nylon) passiveClientRoamed(peer polyamide.Peer) { + n.setPeerReceived(peer, time.Now()) + n.publishLinks() +} + +// setPeerReceived records when peer last received a packet, ignoring older times. +func (n *Nylon) setPeerReceived(peer polyamide.Peer, at time.Time) { + if n.peerReceived == nil { + n.peerReceived = make(map[polyamide.Peer]time.Time) + } + if at.After(n.peerReceived[peer]) { + n.peerReceived[peer] = at + } +} diff --git a/core/nylon_tc.go b/core/nylon_tc.go index aec2c066..20c7807f 100644 --- a/core/nylon_tc.go +++ b/core/nylon_tc.go @@ -1,166 +1,94 @@ package core import ( + "errors" "fmt" - "net/netip" - "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" "google.golang.org/protobuf/proto" ) -const ( - NyProtoId = 8 -) - -// polyamide traffic control for nylon - -func (n *Nylon) InstallTC() { - t := n.Trace - - if n.DBG_trace_tc { - n.Device.InstallFilter(func(dev *device.Device, packet *device.TCElement) (device.TCAction, error) { - if packet.Validate() { // make sure it's an IP packet - peer := packet.FromPeer - if peer == nil { - peer = packet.ToPeer - } - src := packet.GetSrc() - dst := packet.GetDst() - if src.IsValid() && - dst.IsValid() && - peer != nil && - src != netip.IPv4Unspecified() && src != netip.IPv6Unspecified() && - dst != netip.IPv4Unspecified() && dst != netip.IPv6Unspecified() { - t.Submit(fmt.Sprintf("Unhandled TC packet: %v -> %v, peer %s\n", packet.GetSrc(), packet.GetDst(), peer)) - } - } - return device.TcPass, nil - }) +// routeBatch is the traffic-control filter shared by every transport and the TUN. +func (n *Nylon) routeBatch(packets []polyamide.TCElement, decisions []polyamide.TCDecision) { + for i := range packets { + decisions[i] = n.routePacket(packets[i]) } +} - // bounce back packets if using system routing - if n.UseSystemRouting { - n.Device.InstallFilter(func(dev *device.Device, packet *device.TCElement) (device.TCAction, error) { - if packet.Incoming() { - // bounce incoming packets - //dev.Log.Verbosef("BounceFwd packet: %v -> %v", packet.GetSrc(), packet.GetDst()) - return device.TcBounce, nil - } - return device.TcPass, nil - }) - // forward only outgoing packets based on the routing table - n.Device.InstallFilter(func(dev *device.Device, packet *device.TCElement) (device.TCAction, error) { - entry, ok := n.router.Tables.Load().Forward.Lookup(packet.GetDst()) - if ok && !packet.Incoming() { - if entry.Blackhole { - return device.TcDrop, nil - } - packet.ToPeer = entry.Peer - if n.DBG_trace_tc { - t.Submit(fmt.Sprintf("Fwd packet: %v -> %v, via %s\n", packet.GetSrc(), packet.GetDst(), entry.Nh)) - } - return device.TcForward, nil - } - return device.TcPass, nil - }) - } else { - // forward packets based on the routing table - n.Device.InstallFilter(func(dev *device.Device, packet *device.TCElement) (device.TCAction, error) { - entry, ok := n.router.Tables.Load().Forward.Lookup(packet.GetDst()) - if ok { - if entry.Blackhole { - return device.TcDrop, nil - } - packet.ToPeer = entry.Peer - if n.DBG_trace_tc { - t.Submit(fmt.Sprintf("Fwd packet: %v -> %v, via %s\n", packet.GetSrc(), packet.GetDst(), entry.Nh)) - } - return device.TcForward, nil - } - return device.TcPass, nil - }) - - // handle TTL - n.Device.InstallFilter(func(dev *device.Device, packet *device.TCElement) (device.TCAction, error) { - if packet.Incoming() && (packet.GetIPVersion() == 4 || packet.GetIPVersion() == 6) { - // allow traceroute to figure out the route - ttl := packet.GetTTL() - if ttl >= 1 { - ttl-- - packet.DecrementTTL() - } - if ttl == 0 { - if n.DBG_trace_tc { - t.Submit(fmt.Sprintf("TTL Expired: %v -> %v\n", packet.GetSrc(), packet.GetDst())) - } - return device.TcBounce, nil - } - } - return device.TcPass, nil - }) +func (n *Nylon) transportHooks() polyamide.Hooks { + return polyamide.Hooks{ + RouteBatch: n.routeBatch, + Control: func(m polyamide.ControlMessage) { n.handleNylonPacket(m.Payload, m.Endpoint, m.From) }, + DeliverHost: n.deliverTUN, + EndpointLearned: func(peer polyamide.Peer, endpoint polyamide.Endpoint) { + n.Dispatch(func() error { + n.endpointLearned(state.NodeId(peer.ID()), endpoint, peer) + return nil + }) + }, } +} - // handle passive client traffic separately - - // bounce back packets destined for the current node - n.Device.InstallFilter(func(dev *device.Device, packet *device.TCElement) (device.TCAction, error) { - entry, ok := n.router.Tables.Load().Exit.Lookup(packet.GetDst()) - // we should only accept packets destined to us, but not our passive clients - if ok && entry.Nh == n.LocalCfg.Id { +// routePacket drops packets that have no route. +func (n *Nylon) routePacket(packet polyamide.TCElement) polyamide.TCDecision { + tables := n.router.Tables.Load() + if tables == nil { + return polyamide.TCDecision{} + } + if entry, ok := tables.Exit.Lookup(packet.Destination()); ok && entry.Nh == n.LocalCfg.Id { + if n.DBG_trace_tc { + n.Trace.Submit(fmt.Sprintf("Exit: %v -> %v\n", packet.Source(), packet.Destination())) + } + return polyamide.TCDecision{Action: polyamide.TcBounce} + } + if packet.Incoming() { + // The host routing table forwards incoming packets. + if n.UseSystemRouting { + return polyamide.TCDecision{Action: polyamide.TcBounce} + } + packet.DecrementTTL() + if packet.TTL() == 0 { if n.DBG_trace_tc { - t.Submit(fmt.Sprintf("Exit: %v -> %v\n", packet.GetSrc(), packet.GetDst())) + n.Trace.Submit(fmt.Sprintf("TTL Expired: %v -> %v\n", packet.Source(), packet.Destination())) } - //dev.Log.Verbosef("BounceCur packet: %v -> %v", packet.GetSrc(), packet.GetDst()) - return device.TcBounce, nil + return polyamide.TCDecision{Action: polyamide.TcBounce} } - //dev.Log.Verbosef("pass packet: %v -> %v, %v", packet.GetSrc(), packet.GetDst(), entry.Nh) - return device.TcPass, nil - }) - - // handle incoming nylon packets - n.Device.InstallFilter(func(dev *device.Device, packet *device.TCElement) (device.TCAction, error) { - if packet.Incoming() && packet.GetIPVersion() == NyProtoId { - n.handleNylonPacket(packet.Payload(), packet.FromEp, packet.FromPeer) - return device.TcDrop, nil + } + if entry, ok := tables.Forward.Lookup(packet.Destination()); ok { + if entry.Blackhole { + return polyamide.TCDecision{} + } + if n.DBG_trace_tc { + n.Trace.Submit(fmt.Sprintf("Fwd packet: %v -> %v, via %s\n", packet.Source(), packet.Destination(), entry.Nh)) + } + // The transport sends to the peer's default endpoint, which SyncTransport keeps best first. + if links := tables.Links[entry.Nh]; len(links) != 0 { + return polyamide.TCDecision{Action: polyamide.TcForward, To: links[0].Peer} } - return device.TcPass, nil - }) + return polyamide.TCDecision{} + } + if n.DBG_trace_tc { + n.Trace.Submit(fmt.Sprintf("Unhandled TC packet: %v -> %v\n", packet.Source(), packet.Destination())) + } + return polyamide.TCDecision{} } - -func (n *Nylon) SendNylon(pkt *protocol.Ny, endpoint conn.Endpoint, peer *device.Peer) error { +func (n *Nylon) SendNylon(pkt *protocol.Ny, endpoint polyamide.Endpoint, peer polyamide.Peer) error { return n.SendNylonBundle(&protocol.TransportBundle{Packets: []*protocol.Ny{pkt}}, endpoint, peer) } - -func (n *Nylon) SendNylonBundle(pkt *protocol.TransportBundle, endpoint conn.Endpoint, peer *device.Peer) error { - tce := n.Device.NewTCElement() - offset := device.MessageTransportOffsetContent + device.PolyHeaderSize - buf, err := proto.MarshalOptions{ - Deterministic: true, - }.MarshalAppend(tce.Buffer[offset:offset], pkt) +func (n *Nylon) SendNylonBundle(pkt *protocol.TransportBundle, endpoint polyamide.Endpoint, peer polyamide.Peer) error { + data, err := (proto.MarshalOptions{Deterministic: true}).Marshal(pkt) if err != nil { - n.Device.PutMessageBuffer(tce.Buffer) - n.Device.PutTCElement(tce) return err } - tce.InitPacket(NyProtoId, uint16(len(buf)+device.PolyHeaderSize)) - tce.Priority = device.TcHighPriority - - tce.ToEp = endpoint - tce.ToPeer = peer - - // TODO: Optimize? is it worth it? - - tcs := device.NewTCState() - - n.Device.TCBatch([]*device.TCElement{tce}, tcs) - return nil + if peer == nil || peer.Transport() == nil { + return polyamide.ErrInvalidHandle + } + return peer.Transport().SendControl(n.Context, peer, endpoint, data) } -func (n *Nylon) handleNylonPacket(packet []byte, endpoint conn.Endpoint, peer *device.Peer) { - // we need to be careful here, since this function is called on the dataplane +func (n *Nylon) handleNylonPacket(packet []byte, endpoint polyamide.Endpoint, peer polyamide.Peer) { defer func() { err := recover() if err != nil { @@ -171,7 +99,6 @@ func (n *Nylon) handleNylonPacket(packet []byte, endpoint conn.Endpoint, peer *d bundle := &protocol.TransportBundle{} err := proto.Unmarshal(packet, bundle) if err != nil { - // log skipped message n.Log.Debug("Failed to unmarshal packet", "err", err) return } @@ -181,14 +108,15 @@ func (n *Nylon) handleNylonPacket(packet []byte, endpoint conn.Endpoint, peer *d return // not loaded yet } if peer == nil { - n.Log.Debug("dropping nylon packet without an authenticated peer") + n.Log.Debug("dropping nylon packet without a peer") return } - neigh, ok := (*nt)[state.NyPublicKey(peer.GetPublicKey())] + key, valid := peerPublicKey(peer) + neigh, ok := (*nt)[key] + ok = ok && valid if !ok { - // This is expected briefly while a peer is being retired after a config - // change. The packet was authenticated under an obsolete generation. - n.Log.Debug("dropping nylon packet from an unknown peer", "key", peer.GetPublicKey()) + // Retired generations can still deliver callbacks during reconciliation. + n.Log.Debug("dropping nylon packet from an unknown peer", "peer", peer.ID()) return } @@ -212,3 +140,26 @@ func (n *Nylon) handleNylonPacket(packet []byte, endpoint conn.Endpoint, peer *d } } } + +func (n *Nylon) routeTUNPackets(packets [][]byte) error { + elements := make([]polyamide.TCElement, 0, len(packets)) + for _, bytes := range packets { + data, valid := polyamide.IPBytes(bytes) + if valid { + elements = append(elements, polyamide.TCElement{Bytes: data}) + } + } + decisions := make([]polyamide.TCDecision, len(elements)) + n.routeBatch(elements, decisions) + err := polyamide.DispatchBatch(n.Context, elements, decisions) + var host [][]byte + for i, decision := range decisions { + if decision.Action == polyamide.TcBounce { + host = append(host, elements[i].Bytes) + } + } + if len(host) != 0 { + err = errors.Join(err, n.deliverTUN(host)) + } + return err +} diff --git a/core/nylon_transport_test.go b/core/nylon_transport_test.go new file mode 100644 index 00000000..c8d180ee --- /dev/null +++ b/core/nylon_transport_test.go @@ -0,0 +1,378 @@ +package core + +import ( + "bytes" + "context" + "crypto/ecdh" + "crypto/rand" + "io" + "log/slog" + "net/netip" + "testing" + "time" + + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/polyamide/transports/wireguard/adapter" + "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" + "github.com/encodeous/nylon/polyamide/transports/wireguard/tun/tuntest" + "github.com/encodeous/nylon/protocol" + "github.com/encodeous/nylon/state" + "github.com/gaissmai/bart" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "google.golang.org/protobuf/proto" +) + +func routingBackend(t *testing.T) (*adapter.Transport, polyamide.PublicKey) { + t.Helper() + key, err := ecdh.X25519().GenerateKey(rand.Reader) + require.NoError(t, err) + backend, err := adapter.New(adapter.Options{PrivateKey: polyamide.PrivateKey(key.Bytes()), Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, backend.Close()) }) + return backend, polyamide.PublicKey(key.PublicKey().Bytes()) +} +func routingReceive(t *testing.T, packets <-chan []byte) []byte { + t.Helper() + select { + case p := <-packets: + return p + case <-time.After(5 * time.Second): + t.Fatal("timed out receiving routed packet") + return nil + } +} + +func TestNylonForwardsAcrossTransports(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx := context.Background() + var local, remote [2]*adapter.Transport + var localKey, remoteKey [2]polyamide.PublicKey + var localPeer, remotePeer [2]polyamide.Peer + var received [2]chan []byte + var control [2]chan []byte + var endpoints [2]polyamide.Endpoint + n := &Nylon{Context: ctx} + forward := new(bart.Table[RouteTableEntry]) + for i := range local { + local[i], localKey[i] = routingBackend(t) + remote[i], remoteKey[i] = routingBackend(t) + var err error + // Equal IDs on distinct transports must remain distinct handles. + localPeer[i], err = local[i].PreparePeer(ctx, polyamide.PeerConfig{ID: "remote", PublicKey: remoteKey[i]}) + require.NoError(t, err) + remotePeer[i], err = remote[i].PreparePeer(ctx, polyamide.PeerConfig{ID: "nylon", PublicKey: localKey[i]}) + require.NoError(t, err) + destination := netip.AddrFrom4([4]byte{10, 0, 0, byte(i + 1)}) + forward.Insert(netip.PrefixFrom(destination, 32), RouteTableEntry{Nh: state.NodeId([]string{"a", "b"}[i])}) + received[i] = make(chan []byte, 4) + control[i] = make(chan []byte, 4) + } + n.Transports = []polyamide.Transport{local[0], local[1]} + n.router.Tables.Store(&ForwardingTables{Forward: forward, Exit: new(bart.Table[RouteTableEntry]), Links: map[state.NodeId][]Link{"a": {{Peer: localPeer[0]}}, "b": {{Peer: localPeer[1]}}}}) + require.NotEqual(t, localPeer[0], localPeer[1]) + require.Equal(t, localPeer[0].ID(), localPeer[1].ID()) + for i := range local { + require.NoError(t, local[i].Start(ctx, n.transportHooks())) + require.NoError(t, remote[i].Start(ctx, polyamide.Hooks{ + Control: func(m polyamide.ControlMessage) { control[i] <- bytes.Clone(m.Payload) }, + RouteBatch: func(p []polyamide.TCElement, d []polyamide.TCDecision) { + for j := range p { + d[j].Action = polyamide.TcBounce + } + }, + DeliverHost: func(p [][]byte) error { + for _, packet := range p { + received[i] <- bytes.Clone(packet) + } + return nil + }, + })) + } + for i := range local { + for _, pair := range []struct { + from, to *adapter.Transport + peer polyamide.Peer + }{{local[i], remote[i], localPeer[i]}, {remote[i], local[i], remotePeer[i]}} { + endpoint, err := pair.from.PrepareEndpoint(ctx, netip.AddrPortFrom(netip.MustParseAddr("127.0.0.1"), pair.to.ListenPort()).String()) + require.NoError(t, err) + require.NoError(t, pair.from.SetEndpoints(ctx, pair.peer, []polyamide.Endpoint{endpoint})) + if pair.from == local[i] { + endpoints[i] = endpoint + } + } + } + first := tuntest.Ping(netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.1.0.1")) + second := tuntest.Ping(netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.1.0.1")) + require.NoError(t, n.routeTUNPackets([][]byte{first, second})) + require.Equal(t, first, routingReceive(t, received[0])) + require.Equal(t, second, routingReceive(t, received[1])) + // Receive on A and forward through B. Decrement TTL once. + require.NoError(t, remote[0].SendPackets(ctx, []polyamide.OutboundPacket{{Bytes: second, To: remotePeer[0]}})) + expected := bytes.Clone(second) + polyamide.TCElement{Bytes: expected}.DecrementTTL() + require.Equal(t, expected, routingReceive(t, received[1])) + // Forwarding within A retains the same engine buffer and the same TTL policy. + require.NoError(t, remote[0].SendPackets(ctx, []polyamide.OutboundPacket{{Bytes: first, To: remotePeer[0]}})) + expected = bytes.Clone(first) + polyamide.TCElement{Bytes: expected}.DecrementTTL() + require.Equal(t, expected, routingReceive(t, received[0])) + // Control messages use the peer's owning transport. + bundle := &protocol.TransportBundle{Packets: []*protocol.Ny{{}}} + require.NoError(t, n.SendNylonBundle(bundle, nil, localPeer[1])) + data, err := proto.Marshal(bundle) + require.NoError(t, err) + require.Equal(t, data, routingReceive(t, control[1])) + require.Empty(t, control[0]) + // One logical neighbour can choose between both transport instances. + tunablesA := state.DefaultRouterTunables() + tunablesA.MinimumConfidenceWindow = 1 + tunablesB := tunablesA + tunablesB.LinkDeadThreshold = 200 * time.Millisecond + healthA := state.NewEndpoint(endpoints[0].Address(), false, &tunablesA) + healthB := state.NewEndpoint(endpoints[1].Address(), false, &tunablesB) + healthA.Renew() + healthA.UpdatePing(10 * time.Millisecond) + healthB.Renew() + healthB.UpdatePing(time.Millisecond) + n.neighbourLinks = map[state.NodeId][]neighbourLink{"remote": { + {Link: Link{Peer: localPeer[0], Endpoint: endpoints[0]}, health: healthA}, + {Link: Link{Peer: localPeer[1], Endpoint: endpoints[1]}, health: healthB}, + }} + forward = new(bart.Table[RouteTableEntry]) + forward.Insert(netip.MustParsePrefix("10.0.0.1/32"), RouteTableEntry{Nh: "remote"}) + n.router.Tables.Store(&ForwardingTables{Forward: forward, Exit: new(bart.Table[RouteTableEntry])}) + n.publishLinks() + snapshot := n.router.Tables.Load() + require.Same(t, localPeer[1], snapshot.Links["remote"][0].Peer) + require.NoError(t, n.routeTUNPackets([][]byte{first})) + require.Equal(t, first, routingReceive(t, received[1])) + // Expiring B changes only link selection. The logical next hop stays the same. + <-time.After(210 * time.Millisecond) + n.publishLinks() + require.Same(t, forward, n.router.Tables.Load().Forward) + require.Same(t, localPeer[1], snapshot.Links["remote"][0].Peer) + require.NoError(t, n.routeTUNPackets([][]byte{first})) + require.Equal(t, first, routingReceive(t, received[0])) + require.ErrorIs(t, local[0].SendPackets(ctx, []polyamide.OutboundPacket{{Bytes: first, To: localPeer[0], Endpoint: endpoints[1]}}), polyamide.ErrInvalidEndpoint) + // Foreign destinations are rejected before transport submission. + require.ErrorIs(t, local[0].SendPackets(ctx, []polyamide.OutboundPacket{{Bytes: first, To: localPeer[1]}}), polyamide.ErrInvalidHandle) +} + +func TestRuntimeLinksAllowMultipleTransportsAndRetireRemovedBinding(t *testing.T) { + a, _ := routingBackend(t) + b, _ := routingBackend(t) + _, remoteKey := routingBackend(t) + n := testNylonWithPrefixes() + n.Context = context.Background() + n.Transports = []polyamide.Transport{a, b} + n.CentralCfg.Clients = []state.ClientCfg{{NodeCfg: state.NodeCfg{Id: "remote", PubKey: state.NyPublicKey(remoteKey)}}} + n.CentralCfg.Graph = []string{"node, remote"} + configs := []LinkConfig{{Transport: a}, {Transport: b}} + n.linksForNeighbour = func(state.NodeId) []LinkConfig { return configs } + forward := new(bart.Table[RouteTableEntry]) + forward.Insert(netip.MustParsePrefix("10.0.0.1/32"), RouteTableEntry{Nh: "remote"}) + n.router.Tables.Store(&ForwardingTables{Forward: forward, Exit: new(bart.Table[RouteTableEntry])}) + require.NoError(t, n.SyncTransport()) + old := n.transportPeer("remote", a) + other := n.transportPeer("remote", b) + require.NotNil(t, old) + require.NotNil(t, other) + require.Len(t, n.router.Tables.Load().Links["remote"], 2) + snapshot := n.router.Tables.Load() + configs = []LinkConfig{{Transport: b}} + require.NoError(t, n.SyncTransport()) + require.Same(t, other, n.transportPeer("remote", b)) + require.Nil(t, n.transportPeer("remote", a)) + require.Empty(t, a.Peers()) + require.Len(t, b.Peers(), 1) + require.Len(t, snapshot.Links["remote"], 2) + require.Same(t, forward, n.router.Tables.Load().Forward) + require.Same(t, other, n.routePacket(polyamide.TCElement{Bytes: tuntest.Ping(netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.1.0.1"))}).To) + configs = []LinkConfig{{Transport: nil}} + require.ErrorContains(t, n.SyncTransport(), "unregistered transport") + require.Same(t, other, n.transportPeer("remote", b)) +} + +func TestRuntimeLinksCacheEndpointsAndKeepInstanceHealthSeparate(t *testing.T) { + a, _ := routingBackend(t) + b, _ := routingBackend(t) + _, remoteKey := routingBackend(t) + n := testNylonWithPrefixes() + n.Context = context.Background() + n.Transports = []polyamide.Transport{a, b} + n.EndpointResolver = state.NewEndpointResolver(nil) + n.CentralCfg.Routers = append(n.CentralCfg.Routers, state.RouterCfg{NodeCfg: state.NodeCfg{Id: "remote", PubKey: state.NyPublicKey(remoteKey)}}) + n.CentralCfg.Graph = []string{"node, remote"} + n.RouterState.Neighbours = []*state.Neighbour{{Id: "remote"}} + n.linksForNeighbour = func(state.NodeId) []LinkConfig { + return []LinkConfig{{Transport: a, Address: "127.0.0.1:1234"}, {Transport: b, Address: "127.0.0.1:1234"}} + } + n.router.Tables.Store(&ForwardingTables{Forward: new(bart.Table[RouteTableEntry]), Exit: new(bart.Table[RouteTableEntry])}) + require.NoError(t, n.SyncTransport()) + first := n.neighbourLinks["remote"][0] + second := n.neighbourLinks["remote"][1] + require.NotSame(t, first.health, second.health) + require.NoError(t, n.SyncTransport()) + require.Same(t, first.Endpoint, n.neighbourLinks["remote"][0].Endpoint) + require.Same(t, second.Endpoint, n.neighbourLinks["remote"][1].Endpoint) + require.Same(t, first.health, n.receivedLink("remote", first.Endpoint, first.Peer)) + require.Same(t, second.health, n.receivedLink("remote", second.Endpoint, second.Peer)) + first.health.Renew() + n.publishLinks() + require.Len(t, n.router.Tables.Load().Links["remote"], 1) + require.Same(t, first.Peer, n.router.Tables.Load().Links["remote"][0].Peer) +} + +func TestDefaultLinksBindConfiguredEndpointsAndKeepLearnedOnes(t *testing.T) { + a, _ := routingBackend(t) + _, remoteKey := routingBackend(t) + n := testNylonWithPrefixes() + n.Context = context.Background() + n.Transports = []polyamide.Transport{a} + n.EndpointResolver = state.NewEndpointResolver(nil) + n.CentralCfg.Routers = append(n.CentralCfg.Routers, state.RouterCfg{NodeCfg: state.NodeCfg{Id: "remote", PubKey: state.NyPublicKey(remoteKey)}, Endpoints: []string{"127.0.0.1:1234"}}) + n.CentralCfg.Graph = []string{"node, remote"} + // Router-state reconciliation creates health for configured endpoints. + configured := state.NewEndpoint("127.0.0.1:1234", false, &n.RouterTunables) + n.RouterState.Neighbours = []*state.Neighbour{{Id: "remote", Eps: []state.Endpoint{configured}}} + n.router.Tables.Store(&ForwardingTables{Forward: new(bart.Table[RouteTableEntry]), Exit: new(bart.Table[RouteTableEntry])}) + require.NoError(t, n.SyncTransport()) + require.Len(t, n.neighbourLinks["remote"], 1) + link := n.neighbourLinks["remote"][0] + require.Same(t, configured, link.health) + require.Same(t, a, link.Peer.Transport()) + require.Equal(t, "127.0.0.1:1234", link.Endpoint.Address()) + // A ping from an unknown address is learned and survives the next sync. + from, err := a.PrepareEndpoint(n.Context, "127.0.0.1:5678") + require.NoError(t, err) + learned := n.receivedLink("remote", from, link.Peer) + require.NotNil(t, learned) + require.True(t, learned.IsRemote()) + learned.Renew() + require.NoError(t, n.SyncTransport()) + require.Len(t, n.neighbourLinks["remote"], 2) + require.Same(t, configured, n.neighbourLinks["remote"][0].health) + require.Same(t, learned, n.neighbourLinks["remote"][1].health) + require.Len(t, n.RouterState.GetNeighbour("remote").Eps, 2) + // Collecting the learned health also removes its link. + n.RouterState.GetNeighbour("remote").Eps = []state.Endpoint{configured} + n.pruneLinks() + require.Len(t, n.neighbourLinks["remote"], 1) + require.Same(t, configured, n.neighbourLinks["remote"][0].health) +} + +func TestRouteDropsUnroutablePackets(t *testing.T) { + n := testNylonWithPrefixes() + n.router.Tables.Store(&ForwardingTables{Forward: new(bart.Table[RouteTableEntry]), Exit: new(bart.Table[RouteTableEntry])}) + incoming := polyamide.TCElement{Bytes: tuntest.Ping(netip.MustParseAddr("10.0.0.9"), netip.MustParseAddr("10.1.0.1")), From: new(testTransportPeer)} + require.Equal(t, polyamide.TcDrop, n.routePacket(incoming).Action) + n.UseSystemRouting = true + require.Equal(t, polyamide.TcBounce, n.routePacket(incoming).Action) +} + +func TestRouteLeavesEndpointToTransport(t *testing.T) { + n := testNylonWithPrefixes() + peer := new(testTransportPeer) + forward := new(bart.Table[RouteTableEntry]) + forward.Insert(netip.MustParsePrefix("10.0.0.0/24"), RouteTableEntry{Nh: "next"}) + endpoint := testEndpoint("127.0.0.1:1234") + n.router.Tables.Store(&ForwardingTables{Forward: forward, Exit: new(bart.Table[RouteTableEntry]), Links: map[state.NodeId][]Link{"next": {{Peer: peer, Endpoint: endpoint}}}}) + decision := n.routePacket(polyamide.TCElement{Bytes: tuntest.Ping(netip.MustParseAddr("10.0.0.9"), netip.MustParseAddr("10.1.0.1"))}) + require.Equal(t, polyamide.TcForward, decision.Action) + require.Same(t, peer, decision.To) + require.Nil(t, decision.Endpoint) +} + +type testEndpoint string + +func (e testEndpoint) Address() string { return string(e) } + +func TestRuntimeValidation(t *testing.T) { + a := &tunTransport{} + b := &tunTransport{} + host := &testTUN{closed: make(chan struct{})} + require.Error(t, Runtime{Host: host}.validate()) + require.Error(t, Runtime{Transports: []polyamide.Transport{a}}.validate()) + require.Error(t, Runtime{Transports: []polyamide.Transport{a, nil}, Host: host}.validate()) + require.ErrorContains(t, Runtime{Transports: []polyamide.Transport{a, a}, Host: host}.validate(), "duplicate") + require.NoError(t, Runtime{Transports: []polyamide.Transport{a, b}, Host: host}.validate()) +} + +func TestSyncTransportRebindsMovedKeyAndIsolatesFailures(t *testing.T) { + a, localKey := routingBackend(t) + _, keyB := routingBackend(t) + _, keyD := routingBackend(t) + n := testNylonWithPrefixes() + n.Context = context.Background() + n.Transports = []polyamide.Transport{a} + client := func(id state.NodeId, key polyamide.PublicKey) state.ClientCfg { + return state.ClientCfg{NodeCfg: state.NodeCfg{Id: id, PubKey: state.NyPublicKey(key)}} + } + n.CentralCfg.Clients = []state.ClientCfg{client("b", keyB), client("d", keyD)} + n.CentralCfg.Graph = []string{"node, b", "node, d"} + n.router.Tables.Store(&ForwardingTables{Forward: new(bart.Table[RouteTableEntry]), Exit: new(bart.Table[RouteTableEntry])}) + require.NoError(t, n.SyncTransport()) + oldD := n.transportPeer("d", a) + require.NotNil(t, oldD) + // Rename b to c with the same key, and give d this node's own key. + n.CentralCfg.Clients = []state.ClientCfg{client("c", keyB), client("d", localKey)} + n.CentralCfg.Graph = []string{"node, c", "node, d"} + require.ErrorContains(t, n.SyncTransport(), "sync neighbour d") + c := n.transportPeer("c", a) + require.NotNil(t, c) + require.Equal(t, "c", c.ID()) + require.Nil(t, n.transportPeer("b", a)) + // d keeps its previous binding until its config is valid again. + require.Same(t, oldD, n.transportPeer("d", a)) + require.Len(t, a.Peers(), 2) +} + +type segmentedTUN struct { + *testTUN + failed bool +} + +func (h *segmentedTUN) Read(bufs [][]byte, sizes []int, offset int) (int, error) { + if !h.failed { + h.failed = true + return 0, tun.ErrTooManySegments + } + return h.testTUN.Read(bufs, sizes, offset) +} + +func TestTUNReaderSurvivesTooManySegments(t *testing.T) { + ctx, cancel := context.WithCancelCause(context.Background()) + host := &segmentedTUN{testTUN: &testTUN{input: make(chan []byte, 1), closed: make(chan struct{})}} + transport := &tunTransport{packets: make(chan []byte, 1)} + n := &Nylon{Context: ctx, Cancel: cancel, Tun: host, Transports: []polyamide.Transport{transport}, Log: slog.Default()} + table := new(bart.Table[RouteTableEntry]) + table.Insert(netip.MustParsePrefix("10.0.0.2/32"), RouteTableEntry{Nh: "remote"}) + n.router.Tables.Store(&ForwardingTables{Forward: table, Exit: new(bart.Table[RouteTableEntry]), Links: map[state.NodeId][]Link{"remote": {{Peer: &routingPeer{transport: transport, id: "remote"}}}}}) + n.startTUNReader() + defer func() { cancel(context.Canceled); _ = host.Close(); n.tunWorkers.Wait() }() + packet := tuntest.Ping(netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.1")) + host.input <- packet + require.Equal(t, packet, routingReceive(t, transport.packets)) + require.NoError(t, ctx.Err()) +} + +func TestPassiveClientForwardsThroughPeerThatHeardFromIt(t *testing.T) { + a := &tunTransport{packets: make(chan []byte, 1)} + b := &tunTransport{packets: make(chan []byte, 1)} + onA, onB := &routingPeer{transport: a, id: "client"}, &routingPeer{transport: b, id: "client"} + n := testNylonWithPrefixes() + n.Transports = []polyamide.Transport{a, b} + n.CentralCfg.Clients = []state.ClientCfg{{NodeCfg: state.NodeCfg{Id: "client"}}} + n.peerHandles = map[state.NodeId]map[polyamide.Transport]polyamide.Peer{"client": {a: onA, b: onB}} + n.neighbourLinks = map[state.NodeId][]neighbourLink{"client": {{Link: Link{Peer: onA}}, {Link: Link{Peer: onB}}}} + n.router.Tables.Store(&ForwardingTables{Forward: new(bart.Table[RouteTableEntry]), Exit: new(bart.Table[RouteTableEntry])}) + n.publishLinks() + require.Same(t, onA, n.router.Tables.Load().Links["client"][0].Peer) + // The client roams onto transport b. + n.endpointLearned("client", testEndpoint("127.0.0.1:1234"), onB) + require.Same(t, onB, n.router.Tables.Load().Links["client"][0].Peer) + // A stale handle does not change the selection. + n.endpointLearned("client", testEndpoint("127.0.0.1:1234"), &routingPeer{transport: a, id: "client"}) + require.Same(t, onB, n.router.Tables.Load().Links["client"][0].Peer) +} diff --git a/core/nylon_tun.go b/core/nylon_tun.go new file mode 100644 index 00000000..508e4efc --- /dev/null +++ b/core/nylon_tun.go @@ -0,0 +1,131 @@ +package core + +import ( + "errors" + "fmt" + "sync" + + "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" +) + +// Reserve space for headers written by native TUN drivers. +const tunHeadroom = 16 +const maxTUNPacketSize = 65535 + +func (n *Nylon) startTUNReader() { + n.tunWorkers.Add(1) + go func() { + defer n.tunWorkers.Done() + count := n.Tun.BatchSize() + if count <= 0 { + n.Cancel(errors.New("invalid TUN batch size")) + return + } + storage := make([][]byte, count) + sizes := make([]int, count) + packets := make([][]byte, 0, count) + for i := range storage { + storage[i] = make([]byte, tunHeadroom+maxTUNPacketSize) + } + for n.Context.Err() == nil { + read, readErr := n.Tun.Read(storage, sizes, tunHeadroom) + if read < 0 || read > count { + n.Cancel(errors.New("TUN returned invalid packet count")) + return + } + for i, size := range sizes[:read] { + if size < 0 || size > maxTUNPacketSize { + n.Cancel(errors.New("invalid TUN packet size")) + return + } + if size != 0 { + packets = append(packets, storage[i][tunHeadroom:tunHeadroom+size]) + } + } + if len(packets) != 0 { + if err := n.routeTUNPackets(packets); err != nil && n.Context.Err() == nil { + n.Log.Warn("failed to submit TUN packets", "err", err) + } + } + clear(packets) + packets = packets[:0] + if readErr != nil { + // Like upstream WireGuard, a read with too many segments only drops packets. + if errors.Is(readErr, tun.ErrTooManySegments) { + n.Log.Debug("dropped packets from multi-segment TUN read", "err", readErr) + continue + } + if n.Context.Err() == nil { + n.Cancel(fmt.Errorf("read TUN packets: %w", readErr)) + } + return + } + } + }() +} + +// watchTUNEvents passes host MTU changes to the transports and logs up and down +// events. Transports keep running while the host interface is down. +func (n *Nylon) watchTUNEvents() { + host, ok := n.Tun.(interface{ Events() <-chan tun.Event }) + if !ok { + return + } + n.tunWorkers.Add(1) + go func() { + defer n.tunWorkers.Done() + for event := range host.Events() { + if event&tun.EventMTUUpdate != 0 { + n.applyHostMTU() + } + if event&tun.EventUp != 0 { + n.Log.Info("host interface is up", "name", n.Interface) + } + if event&tun.EventDown != 0 { + n.Log.Warn("host interface is down", "name", n.Interface) + } + } + }() +} + +func (n *Nylon) applyHostMTU() { + mtu, err := n.Tun.MTU() + if err != nil || mtu <= 0 { + n.Log.Error("failed to read host MTU", "mtu", mtu, "err", err) + return + } + for _, transport := range n.Transports { + transport.SetHostMTU(mtu) + } +} + +func (n *Nylon) deliverTUN(packets [][]byte) error { + // Use Nylon-owned buffers for the headers written by the TUN. + pooled := make([]*[]byte, len(packets)) + buffers := make([][]byte, len(packets)) + for i, packet := range packets { + pooled[i] = getHostBuffer(tunHeadroom + len(packet)) + buffers[i] = *pooled[i] + copy(buffers[i][tunHeadroom:], packet) + } + // Implementations disagree on the count they return: Linux reports bytes. + _, err := n.Tun.Write(buffers, tunHeadroom) + for _, buffer := range pooled { + hostBuffers.Put(buffer) + } + return err +} + +var hostBuffers sync.Pool + +// getHostBuffer returns a pooled buffer of length size, with at least +// minHostBufferCap capacity. A pooled buffer that is too small is replaced. +func getHostBuffer(size int) *[]byte { + buffer, _ := hostBuffers.Get().(*[]byte) + if buffer == nil || cap(*buffer) < size { + buffer = new([]byte) + *buffer = make([]byte, size, max(size, minHostBufferCap)) + } + *buffer = (*buffer)[:size] + return buffer +} diff --git a/core/nylon_tun_linux.go b/core/nylon_tun_linux.go new file mode 100644 index 00000000..66758185 --- /dev/null +++ b/core/nylon_tun_linux.go @@ -0,0 +1,5 @@ +package core + +// Linux GRO coalesces later packets into the spare capacity of a host buffer, +// so each one can hold a full TUN packet. +const minHostBufferCap = tunHeadroom + maxTUNPacketSize diff --git a/core/nylon_tun_other.go b/core/nylon_tun_other.go new file mode 100644 index 00000000..63cde58c --- /dev/null +++ b/core/nylon_tun_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package core + +// Other platforms do not coalesce packets, so host buffers start large enough +// for a packet at the common Ethernet MTU and grow when a packet is larger. +const minHostBufferCap = tunHeadroom + 1500 diff --git a/core/nylon_tun_test.go b/core/nylon_tun_test.go new file mode 100644 index 00000000..194716f9 --- /dev/null +++ b/core/nylon_tun_test.go @@ -0,0 +1,183 @@ +package core + +import ( + "context" + "io" + "log/slog" + "sync" + "testing" + "time" + + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/polyamide/transports/wireguard/tun/tuntest" + "github.com/encodeous/nylon/state" + "github.com/gaissmai/bart" + "github.com/stretchr/testify/require" + "net/netip" +) + +type testTUN struct { + input chan []byte + output chan []byte + closed chan struct{} + once sync.Once +} + +func (h *testTUN) Read(bufs [][]byte, sizes []int, offset int) (int, error) { + select { + case p := <-h.input: + sizes[0] = copy(bufs[0][offset:], p) + return 1, nil + case <-h.closed: + return 0, io.EOF + } +} +func (h *testTUN) Write(bufs [][]byte, offset int) (int, error) { + if offset < 10 { + return 0, io.ErrShortBuffer + } + for _, p := range bufs { + h.output <- append([]byte(nil), p[offset:]...) + } + return len(bufs), nil +} +func (*testTUN) MTU() (int, error) { return 1420, nil } +func (*testTUN) Name() (string, error) { return "test-host", nil } +func (*testTUN) BatchSize() int { return 1 } +func (h *testTUN) Close() error { h.once.Do(func() { close(h.closed) }); return nil } + +type tunTransport struct { + polyamide.Transport + packets chan []byte + closed bool +} + +func (t *tunTransport) SendPackets(_ context.Context, packets []polyamide.OutboundPacket) error { + for _, packet := range packets { + t.packets <- append([]byte(nil), packet.Bytes...) + } + return nil +} +func (t *tunTransport) Close() error { t.closed = true; return nil } + +func TestNylonOwnsTUNIOAndShutdown(t *testing.T) { + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(context.Canceled) + host := &testTUN{input: make(chan []byte, 1), output: make(chan []byte, 1), closed: make(chan struct{})} + transport := &tunTransport{packets: make(chan []byte, 1)} + secondary := &tunTransport{packets: make(chan []byte, 1)} + n := &Nylon{Context: ctx, Cancel: cancel, Tun: host, Transports: []polyamide.Transport{transport, secondary}, Log: slog.Default()} + n.startTUNReader() + defer func() { cancel(context.Canceled); require.NoError(t, n.cleanupTransport()) }() + packet := tuntest.Ping(netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.1")) + peer := &routingPeer{transport: transport, id: "remote"} + table := new(bart.Table[RouteTableEntry]) + table.Insert(netip.MustParsePrefix("10.0.0.2/32"), RouteTableEntry{Nh: "remote"}) + n.router.Tables.Store(&ForwardingTables{Forward: table, Exit: new(bart.Table[RouteTableEntry]), Links: map[state.NodeId][]Link{"remote": {{Peer: peer}}}}) + host.input <- packet + select { + case got := <-transport.packets: + require.Equal(t, packet, got) + case <-time.After(time.Second): + t.Fatal("Nylon did not submit host packet") + } + require.NoError(t, n.deliverTUN([][]byte{packet})) + require.Equal(t, packet, <-host.output) + // Closing a transport cannot close the host interface. + require.NoError(t, transport.Close()) + select { + case <-host.closed: + t.Fatal("transport closed the host") + default: + } + cancel(context.Canceled) + require.NoError(t, n.cleanupTransport()) + require.True(t, transport.closed) + require.True(t, secondary.closed) + select { + case <-host.closed: + default: + t.Fatal("Nylon did not close the host") + } +} + +type routingPeer struct { + transport polyamide.Transport + id string +} + +func (p *routingPeer) Transport() polyamide.Transport { return p.transport } +func (p *routingPeer) ID() string { return p.id } +func (*routingPeer) PublicKey() polyamide.PublicKey { return polyamide.PublicKey{} } + +func TestTUNRoutesEachDestinationToItsTransport(t *testing.T) { + a := &tunTransport{packets: make(chan []byte, 4)} + b := &tunTransport{packets: make(chan []byte, 4)} + host := &testTUN{output: make(chan []byte, 4)} + n := &Nylon{Context: context.Background(), Transports: []polyamide.Transport{a, b}, Tun: host} + forward := new(bart.Table[RouteTableEntry]) + exit := new(bart.Table[RouteTableEntry]) + first := &routingPeer{transport: a, id: "a"} + second := &routingPeer{transport: b, id: "b"} + forward.Insert(netip.MustParsePrefix("10.0.0.1/32"), RouteTableEntry{Nh: "a"}) + forward.Insert(netip.MustParsePrefix("10.0.0.2/32"), RouteTableEntry{Nh: "b"}) + forward.Insert(netip.MustParsePrefix("10.0.0.3/32"), RouteTableEntry{Blackhole: true}) + exit.Insert(netip.MustParsePrefix("10.0.0.4/32"), RouteTableEntry{}) + n.router.Tables.Store(&ForwardingTables{Forward: forward, Exit: exit, Links: map[state.NodeId][]Link{"a": {{Peer: first}}, "b": {{Peer: second}}}}) + packet := func(dest string) []byte { + return tuntest.Ping(netip.MustParseAddr(dest), netip.MustParseAddr("10.1.0.1")) + } + one, two, drop, local, unknown := packet("10.0.0.1"), packet("10.0.0.2"), packet("10.0.0.3"), packet("10.0.0.4"), packet("10.0.0.5") + require.NoError(t, n.routeTUNPackets([][]byte{one, two, drop, local, unknown})) + require.Equal(t, one, <-a.packets) + require.Equal(t, two, <-b.packets) + require.Equal(t, local, <-host.output) + require.Empty(t, a.packets) + require.Empty(t, b.packets) + // Changing the published forwarding table changes the backend immediately. + next := new(bart.Table[RouteTableEntry]) + next.Insert(netip.MustParsePrefix("10.0.0.1/32"), RouteTableEntry{Nh: "b"}) + n.router.Tables.Store(&ForwardingTables{Forward: next, Exit: exit, Links: map[state.NodeId][]Link{"b": {{Peer: second}}}}) + require.NoError(t, n.routeTUNPackets([][]byte{one})) + require.Equal(t, one, <-b.packets) + require.Empty(t, a.packets) +} + +// byteCountTUN reports bytes written, like the Linux TUN. +type byteCountTUN struct { + *testTUN + minCap int +} + +func (h *byteCountTUN) Write(bufs [][]byte, offset int) (int, error) { + total := 0 + h.minCap = -1 + for _, b := range bufs { + total += len(b) - offset + if h.minCap < 0 || cap(b) < h.minCap { + h.minCap = cap(b) + } + } + _, err := h.testTUN.Write(bufs, offset) + return total, err +} + +func TestDeliverTUNAcceptsByteCounts(t *testing.T) { + host := &byteCountTUN{testTUN: &testTUN{output: make(chan []byte, 2)}} + n := &Nylon{Tun: host} + packet := tuntest.Ping(netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.1")) + require.NoError(t, n.deliverTUN([][]byte{packet, packet})) + require.Equal(t, packet, <-host.output) + // On Linux, spare capacity lets GRO coalesce the packets that follow. + require.GreaterOrEqual(t, host.minCap, minHostBufferCap) +} + +func TestHostBuffersGrowForLargerPackets(t *testing.T) { + small := getHostBuffer(10) + require.Len(t, *small, 10) + require.GreaterOrEqual(t, cap(*small), minHostBufferCap) + hostBuffers.Put(small) + large := getHostBuffer(minHostBufferCap + 1) + require.Len(t, *large, minHostBufferCap+1) + hostBuffers.Put(large) +} diff --git a/core/nylon_wireguard.go b/core/nylon_wireguard.go index c514706f..d32bf974 100644 --- a/core/nylon_wireguard.go +++ b/core/nylon_wireguard.go @@ -1,65 +1,52 @@ package core import ( - "bufio" - "cmp" - "encoding/hex" "errors" "fmt" "net/netip" "runtime" "slices" - "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/state" ) -func (n *Nylon) initWireGuard() error { - dev, tdev, itfName, err := NewWireGuardDevice(n) - if err != nil { - return err +func (n *Nylon) initTransport() error { + if len(n.Transports) == 0 { + rt, err := newDefaultRuntime(n) + if err != nil { + return err + } + if err := rt.validate(); err != nil { + rt.close() + return err + } + n.useRuntime(rt) } - - err = dev.Up() + name, err := n.Tun.Name() if err != nil { return err } - - n.Device = dev - n.Tun = tdev - n.Interface = itfName - - n.InstallTC() - n.Log.Info("installed nylon traffic control filter for polysock") - - dev.IpcHandler["get=nylon\n"] = func(writer *bufio.ReadWriter) error { - return HandleNylonIPC(n, writer) + n.Interface = name + // Nodes that fail to sync are retried by the reconciliation task below. + if err := n.SyncTransport(); err != nil { + n.Log.Warn("initial transport sync incomplete; will retry", "err", err) } - - // TODO: fully convert to code-based api - err = dev.IpcSet( - fmt.Sprintf( - `private_key=%s -listen_port=%d -`, - hex.EncodeToString(n.Key[:]), - n.Port, - ), - ) - if err != nil { - return fmt.Errorf("failed to configure wg device: %v", err) + n.applyHostMTU() + for _, transport := range n.Transports { + if err := transport.Start(n.Context, n.transportHooks()); err != nil { + return err + } } - - // add peers - err = n.SyncWireGuard() - if err != nil { - return err + if n.wireGuard != nil { + n.wireGuard.serve(n.Context, n.Log) } - + n.startTUNReader() + n.watchTUNEvents() // configure system networking // run pre-up commands + n.systemConfigured = true for _, cmd := range n.PreUp { err = ExecSplit(n.Log, cmd) if err != nil { @@ -69,7 +56,7 @@ listen_port=%d if !n.NoNetConfigure && !n.NoTun { for _, addr := range n.GetRouter(n.LocalCfg.Id).Addresses { - err := ConfigureAlias(n.Log, itfName, addr) + err := ConfigureAlias(n.Log, n.Interface, addr) if err != nil { n.Log.Error("failed to configure alias", "err", err) } else if !slices.Contains(n.AppliedSystem.Aliases, addr) { @@ -77,7 +64,7 @@ listen_port=%d } } - err = InitInterface(n.Log, itfName) + err = InitInterface(n.Log, n.Interface) if err != nil { return err } @@ -102,136 +89,54 @@ listen_port=%d return nil } -func (n *Nylon) cleanupWireGuard() error { - // remove routes - for _, route := range n.AppliedSystem.Routes { - err := RemoveRoute(n.Log, n.Tun, n.Interface, route) - if err != nil { - n.Log.Error("failed to remove route", "err", err) - } - } - for _, addr := range n.AppliedSystem.Aliases { - err := RemoveAlias(n.Log, n.Interface, addr) - if err != nil { - n.Log.Error("failed to remove alias", "err", err) +func (n *Nylon) cleanupTransport() error { + if n.systemConfigured { + for _, route := range n.AppliedSystem.Routes { + if err := RemoveRoute(n.Log, n.Tun, n.Interface, route); err != nil { + n.Log.Error("failed to remove route", "err", err) + } } - } - // run pre-down commands - for _, cmd := range n.PreDown { - err := ExecSplit(n.Log, cmd) - if err != nil { - n.Log.Error("failed to run pre-down command", "err", err) + for _, addr := range n.AppliedSystem.Aliases { + if err := RemoveAlias(n.Log, n.Interface, addr); err != nil { + n.Log.Error("failed to remove alias", "err", err) + } } - } - err := CleanupWireGuardDevice(n) - if err != nil { - return err - } - // run post-down commands - for _, cmd := range n.PostDown { - err = ExecSplit(n.Log, cmd) - if err != nil { - n.Log.Error("failed to run post-down command", "err", err) + for _, cmd := range n.PreDown { + if err := ExecSplit(n.Log, cmd); err != nil { + n.Log.Error("failed to run pre-down command", "err", err) + } } } - return nil -} - -func (n *Nylon) SyncWireGuard() error { - if n.Device == nil { - return nil + var err error + if n.wireGuard != nil { + err = errors.Join(err, n.wireGuard.close()) } - if n.AppliedSystem.Peers == nil { - n.AppliedSystem.Peers = make(map[state.NodeId]state.NyPublicKey) + if n.Tun != nil { + err = errors.Join(err, n.Tun.Close()) } - - desired := make(map[state.NodeId]state.NyPublicKey) - for _, peer := range n.GetPeers(n.LocalCfg.Id) { - ncfg := n.GetNode(peer) - desired[peer] = ncfg.PubKey + n.tunWorkers.Wait() + for _, transport := range n.Transports { + err = errors.Join(err, transport.Close()) } - - // Prepare every desired peer before removing any old peer. In particular, - // public-key rotation must keep the old peer alive until the forwarding - // table has been rebound to the replacement. - for _, peer := range slices.Sorted(slices.Values(n.GetPeers(n.LocalCfg.Id))) { - ncfg := n.GetNode(peer) - wgPeer := n.Device.LookupPeer(device.NoisePublicKey(ncfg.PubKey)) - if wgPeer == nil { - n.Log.Debug("adding", "peer", peer) - var err error - wgPeer, err = n.Device.NewPeer(device.NoisePublicKey(ncfg.PubKey)) - if err != nil { - return err + if n.systemConfigured { + for _, cmd := range n.PostDown { + if e := ExecSplit(n.Log, cmd); e != nil { + n.Log.Error("failed to run post-down command", "err", e) } - wgPeer.Start() - } - if n.IsClient(peer) { - wgPeer.SetPreferRoaming(true) } } - - if err := n.syncWireGuardEndpoints(); err != nil { - return err - } - - // The forwarding table caches concrete peers for the one-lookup hot path. - // Rebind every entry before stopping peers from the previous generation. - n.rebindForwardingPeers() - - desiredKeys := make(map[state.NyPublicKey]struct{}, len(desired)) - for _, key := range desired { - desiredKeys[key] = struct{}{} - } - for _, wgPeer := range n.Device.GetPeers() { - key := state.NyPublicKey(wgPeer.GetPublicKey()) - if _, ok := desiredKeys[key]; ok { - continue - } - n.Log.Debug("removing obsolete WireGuard peer", "key", key) - n.Device.RemovePeer(wgPeer.GetPublicKey()) - } - - n.AppliedSystem.Peers = desired - return nil + return err } -func (n *Nylon) syncWireGuardEndpoints() error { - if n.Device == nil { - return nil - } - dev := n.Device - - // configure endpoints - for _, peer := range slices.Sorted(slices.Values(n.GetPeers(n.LocalCfg.Id))) { - if n.IsClient(peer) { - continue - } - pcfg := n.GetRouter(peer) - nhNeigh := n.RouterState.GetNeighbour(peer) - eps := make([]conn.Endpoint, 0) - - if nhNeigh != nil { - links := slices.Clone(nhNeigh.Eps) - slices.SortStableFunc(links, func(a, b state.Endpoint) int { - return cmp.Compare(a.Metric(), b.Metric()) - }) - for _, ep := range links { - nep, err := ep.AsNylonEndpoint().GetWgEndpoint(n.Device, n.EndpointResolver) - if err != nil { - continue - } - eps = append(eps, nep) - } - } - - wgPeer := dev.LookupPeer(device.NoisePublicKey(pcfg.PubKey)) - if wgPeer != nil { - wgPeer.SetEndpoints(eps) - } +func (n *Nylon) transportPeer(id state.NodeId, transport polyamide.Transport) polyamide.Peer { + return n.peerHandles[id][transport] +} +func peerPublicKey(peer polyamide.Peer) (state.NyPublicKey, bool) { + var key state.NyPublicKey + if peer == nil { + return key, false } - - return nil + return state.NyPublicKey(peer.PublicKey()), true } func (n *Nylon) SyncSystemState() error { @@ -318,3 +223,20 @@ func (n *Nylon) syncSystemRoutes() error { n.AppliedSystem.Routes = applied return syncErr } + +// defaultTransport carries configured endpoints when the runtime supplies no links. +// The platform runtime uses WireGuard. +func (n *Nylon) defaultTransport() polyamide.Transport { + if len(n.Transports) == 0 { + return nil + } + return n.Transports[0] +} + +func (n *Nylon) transportPeers() []polyamide.PeerStats { + var peers []polyamide.PeerStats + for _, transport := range n.Transports { + peers = append(peers, transport.Peers()...) + } + return peers +} diff --git a/core/router.go b/core/router.go index a818bec3..aebf19cc 100644 --- a/core/router.go +++ b/core/router.go @@ -4,9 +4,9 @@ import ( "net/netip" "time" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" "github.com/gaissmai/bart" "go4.org/netipx" + "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" "github.com/encodeous/nylon/log" @@ -17,7 +17,6 @@ import ( type RouteTableEntry struct { Nh state.NodeId - Peer *device.Peer Blackhole bool } @@ -25,7 +24,8 @@ type ForwardingTables struct { // Forward contains the full routing table. Forward *bart.Table[RouteTableEntry] // Exit contains only routes to services hosted on this node. - Exit *bart.Table[RouteTableEntry] + Exit *bart.Table[RouteTableEntry] + Links map[state.NodeId][]Link } func (n *Nylon) GetNeighIO(neigh state.NodeId) *IOPending { @@ -115,23 +115,20 @@ func (n *Nylon) TableInsertRoute(prefix netip.Prefix, route state.SelRoute) { Blackhole: true, }) ne.Delete(prefix) - n.router.Tables.Store(&ForwardingTables{Forward: nf, Exit: ne}) + n.router.Tables.Store(&ForwardingTables{Forward: nf, Exit: ne, Links: n.forwardingLinks()}) return } - peer := n.Device.LookupPeer(device.NoisePublicKey(n.GetNode(nh).PubKey)) nf.Insert(prefix, RouteTableEntry{ - Nh: nh, - Peer: peer, + Nh: nh, }) if route.Nh == n.LocalCfg.Id { ne.Insert(prefix, RouteTableEntry{ - Nh: nh, - Peer: peer, + Nh: nh, }) } else { ne.Delete(prefix) } - n.router.Tables.Store(&ForwardingTables{Forward: nf, Exit: ne}) + n.router.Tables.Store(&ForwardingTables{Forward: nf, Exit: ne, Links: n.forwardingLinks()}) } func (n *Nylon) TableDeleteRoute(prefix netip.Prefix) { @@ -140,50 +137,14 @@ func (n *Nylon) TableDeleteRoute(prefix netip.Prefix) { ne := tables.Exit.Clone() nf.Delete(prefix) ne.Delete(prefix) - n.router.Tables.Store(&ForwardingTables{Forward: nf, Exit: ne}) + n.router.Tables.Store(&ForwardingTables{Forward: nf, Exit: ne, Links: n.forwardingLinks()}) } -func (n *Nylon) rebindForwardingPeers() { - if n.Device == nil { - return - } - +func (n *Nylon) publishLinks() { tables := n.router.Tables.Load() - if tables == nil { - return - } - - peers := make(map[state.NodeId]*device.Peer) - for _, node := range n.CentralCfg.GetNodes() { - peers[node.Id] = n.Device.LookupPeer(device.NoisePublicKey(node.PubKey)) - } - - forward, forwardChanged := rebindRouteTablePeers(tables.Forward, peers) - exit, exitChanged := rebindRouteTablePeers(tables.Exit, peers) - if forwardChanged || exitChanged { - n.router.Tables.Store(&ForwardingTables{Forward: forward, Exit: exit}) - } -} - -func rebindRouteTablePeers(table *bart.Table[RouteTableEntry], peers map[state.NodeId]*device.Peer) (*bart.Table[RouteTableEntry], bool) { - next := table - changed := false - for prefix, entry := range table.All() { - if entry.Blackhole { - continue - } - peer := peers[entry.Nh] - if entry.Peer == peer { - continue - } - if !changed { - next = table.Clone() - changed = true - } - entry.Peer = peer - next.Insert(prefix, entry) + if tables != nil { + n.router.Tables.Store(&ForwardingTables{Forward: tables.Forward, Exit: tables.Exit, Links: n.forwardingLinks()}) } - return next, changed } type IOPending struct { @@ -403,6 +364,12 @@ func (n *Nylon) routerHandleSeqnoRequest(neigh state.NodeId, pkt *protocol.Ny_Se return nil } +// bundleEntrySize includes the field tag and length prefix of a repeated TransportBundle entry. +func bundleEntrySize(m *protocol.Ny) int { + size := proto.Size(m) + return protowire.SizeTag(1) + protowire.SizeBytes(size) +} + func (n *Nylon) flushIO() error { for _, neigh := range n.RouterState.Neighbours { // TODO, investigate effect of packet loss on control messages @@ -412,7 +379,15 @@ func (n *Nylon) flushIO() error { continue } if best != nil && best.IsActive() { - peer := n.Device.LookupPeer(device.NoisePublicKey(n.GetNode(neigh.Id).PubKey)) + link, ok := n.bestLink(neigh.Id) + if !ok { + continue + } + // Bundles must fit both the path MTU and the transport's control capacity. + limit := n.SafeMTU + if capacity, err := link.Peer.Transport().ControlMTU(link.Peer, link.Endpoint); err == nil { + limit = min(limit, capacity) + } for { bundle := &protocol.TransportBundle{} tLength := 0 @@ -430,24 +405,24 @@ func (n *Nylon) flushIO() error { HopCount: uint32(nio.SeqnoReq[seqR].V2), }, }} - if tLength != 0 && tLength+proto.Size(req) >= n.SafeMTU { + if tLength != 0 && tLength+bundleEntrySize(req) >= limit { goto send } delete(nio.SeqnoReq, seqR) bundle.Packets = append(bundle.Packets, req) - tLength += proto.Size(req) + tLength += bundleEntrySize(req) } for id, update := range nio.Updates { req := &protocol.Ny{Type: &protocol.Ny_RouteOp{ RouteOp: update, }} - if tLength != 0 && tLength+proto.Size(req) >= n.SafeMTU { + if tLength != 0 && tLength+bundleEntrySize(req) >= limit { goto send } delete(nio.Updates, id) bundle.Packets = append(bundle.Packets, req) - tLength += proto.Size(req) + tLength += bundleEntrySize(req) } for prefix := range nio.Acks { @@ -457,21 +432,21 @@ func (n *Nylon) flushIO() error { Prefix: prefixBytes, }, }} - if tLength != 0 && tLength+proto.Size(req) >= n.SafeMTU { + if tLength != 0 && tLength+bundleEntrySize(req) >= limit { goto send } delete(nio.Acks, prefix) bundle.Packets = append(bundle.Packets, req) - tLength += proto.Size(req) + tLength += bundleEntrySize(req) } if tLength == 0 { break } send: - err := n.SendNylonBundle(bundle, nil, peer) - if err != nil { - return err + // Control messages are best effort. A failed send must not stop the node. + if err := n.SendNylonBundle(bundle, link.Endpoint, link.Peer); err != nil { + n.Log.Debug("failed to send control bundle", "neighbour", neigh.Id, "err", err) } } } diff --git a/core/sys_darwin.go b/core/sys_darwin.go index 06f7ec98..d4393d23 100644 --- a/core/sys_darwin.go +++ b/core/sys_darwin.go @@ -5,8 +5,8 @@ import ( "net" "net/netip" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/polyamide/transports/wireguard/ipc" - "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" ) func InitUAPI(logger *slog.Logger, itfName string) (net.Listener, error) { @@ -63,7 +63,7 @@ func PrefixToMaskString(p netip.Prefix) string { return net.IP(mask).String() } -func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { +func ConfigureRoute(logger *slog.Logger, dev polyamide.HostDevice, itfName string, route netip.Prefix) error { if route.Addr().Is6() { return Exec(logger, "/sbin/route", "-n", "add", "-inet6", route.String(), "-interface", itfName) } else { @@ -73,7 +73,7 @@ func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route n } } -func RemoveRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { +func RemoveRoute(logger *slog.Logger, dev polyamide.HostDevice, itfName string, route netip.Prefix) error { if route.Addr().Is6() { return Exec(logger, "/sbin/route", "-n", "delete", "-inet6", route.String(), "-interface", itfName) } else { diff --git a/core/sys_linux.go b/core/sys_linux.go index e096f38b..f0bb6887 100644 --- a/core/sys_linux.go +++ b/core/sys_linux.go @@ -5,8 +5,8 @@ import ( "net" "net/netip" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/polyamide/transports/wireguard/ipc" - "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" "github.com/encodeous/nylon/state" ) @@ -39,10 +39,10 @@ func RemoveAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { return Exec(logger, "ip", "addr", "del", state.AddrToPrefix(addr).String(), "dev", ifName) } -func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { +func ConfigureRoute(logger *slog.Logger, dev polyamide.HostDevice, itfName string, route netip.Prefix) error { return Exec(logger, "ip", "route", "add", route.String(), "dev", itfName) } -func RemoveRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { +func RemoveRoute(logger *slog.Logger, dev polyamide.HostDevice, itfName string, route netip.Prefix) error { return Exec(logger, "ip", "route", "del", route.String(), "dev", itfName) } diff --git a/core/sys_physical.go b/core/sys_physical.go index 4bbebaf0..9210648f 100644 --- a/core/sys_physical.go +++ b/core/sys_physical.go @@ -4,87 +4,49 @@ package core import ( "fmt" + "net" "runtime" - "strings" - "github.com/encodeous/nylon/log" - "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/polyamide/transports/wireguard/device" "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" ) -func NewWireGuardDevice(n *Nylon) (dev *device.Device, tunDevice tun.Device, realItf string, err error) { - itfName := n.InterfaceName // attempt to name the interface - +func newDefaultRuntime(n *Nylon) (Runtime, error) { + name := n.InterfaceName if runtime.GOOS == "darwin" && !n.NoTun { - itfName = "utun" + name = "utun" } - - var tdev tun.Device + var host tun.Device + var err error if n.NoTun { - tdev = tun.NewDummyDevice(itfName) + host = tun.NewDummyDevice(name) } else { - tdev, err = tun.CreateTUN(itfName, device.DefaultMTU) + host, err = tun.CreateTUN(name, device.DefaultMTU) if err != nil { - return nil, nil, "", fmt.Errorf("failed to create TUN: %v. Check if an interface with the name nylon exists already", err) + return Runtime{}, fmt.Errorf("failed to create TUN: %w. Check if an interface with the name %s exists already", err, name) } } - realInterfaceName, err := tdev.Name() + transport, err := NewWireGuardTransport(n, nil) + if err != nil { + host.Close() + return Runtime{}, err + } + actual, err := host.Name() + var listener net.Listener if err == nil { - itfName = realInterfaceName + listener, err = InitUAPI(n.Log, actual) } - - wgLog := n.Log.With("module", log.ScopePolyamide) - - // setup WireGuard - dev = device.NewDevice(tdev, conn.NewDefaultBind(), &device.Logger{ - Verbosef: func(format string, args ...any) { - if n.DBG_log_wireguard { - wgLog.Debug(fmt.Sprintf(format, args...)) - } - }, - Errorf: func(format string, args ...any) { - if strings.Contains(format, "Failed to send PolySock packets") { - return - } - wgLog.Error(fmt.Sprintf(format, args...)) - }, - }) - - // start uapi for wg command - n.wgUapi, err = InitUAPI(n.Log, itfName) if err != nil { - return nil, nil, "", err + host.Close() + transport.Close() + return Runtime{}, err } - - if n.wgUapi != nil { - go func() { - for n.Context.Err() == nil { - accept, err := n.wgUapi.Accept() - if err != nil { - n.Log.Debug(err.Error()) - continue - } - go dev.IpcHandle(accept) - } - }() - } - + n.wireGuard = &legacyWireGuard{transport: transport, listener: listener} if n.NoTun { - n.Log.Info("Created userspace-only WireGuard device", "name", itfName) + n.Log.Info("Created userspace-only WireGuard device", "name", actual) } else { - n.Log.Info("Created WireGuard interface", "name", itfName) - } - return dev, tdev, itfName, nil -} - -func CleanupWireGuardDevice(n *Nylon) error { - n.Device.Close() - if n.wgUapi != nil { - err := n.wgUapi.Close() - if err != nil { - return err - } + n.Log.Info("Created WireGuard interface", "name", actual) } - return nil + return Runtime{Transports: []polyamide.Transport{transport}, Host: host}, nil } diff --git a/core/sys_virtual.go b/core/sys_virtual.go index 86655c2a..fa509b13 100644 --- a/core/sys_virtual.go +++ b/core/sys_virtual.go @@ -4,75 +4,30 @@ package core import ( "fmt" - "strings" - "github.com/encodeous/nylon/log" - "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" - "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" - "github.com/encodeous/nylon/state" + "github.com/encodeous/nylon/polyamide/transports/wireguard/adapter" ) -type VirtualNet interface { - Bind(node state.NodeId) conn.Bind - Tun(node state.NodeId) tun.Device +// The integration environment supplies a complete runtime. Nylon does not know +// which protocol, sockets, or bind implementation that transport uses. +type virtualRuntimeFactory interface { + NewRuntime(*Nylon) (Runtime, error) } -func NewWireGuardDevice(n *Nylon) (dev *device.Device, tunDevice tun.Device, realItf string, err error) { - x := n.AuxConfig["vnet"] - if x == nil { - return nil, nil, "", fmt.Errorf("expected aux config \"vnet\", but it was not present") +func newDefaultRuntime(n *Nylon) (Runtime, error) { + factory, ok := n.AuxConfig["vnet"].(virtualRuntimeFactory) + if !ok { + return Runtime{}, fmt.Errorf("expected aux config vnet runtime factory") } - vn := x.(VirtualNet) - - itfName := "nylon-vn" - - bind := vn.Bind(n.LocalCfg.Id) - var tdev tun.Device - if n.NoTun { - tdev = tun.NewDummyDevice(itfName) - } else { - tdev = vn.Tun(n.LocalCfg.Id) - } - - wgLog := n.Log.With("module", log.ScopePolyamide) - - // setup WireGuard - dev = device.NewDevice(tdev, bind, &device.Logger{ - Verbosef: func(format string, args ...any) { - if n.DBG_log_wireguard { - wgLog.Debug(fmt.Sprintf(format, args...)) - } - }, - Errorf: func(format string, args ...any) { - if strings.Contains(format, "Failed to send PolySock packets") { - return - } - wgLog.Error(fmt.Sprintf(format, args...)) - }, - }) - - if n.NoTun { - n.Log.Info("Created userspace-only WireGuard device", "name", itfName) - } else { - n.Log.Info("Created WireGuard interface", "name", itfName) - } - return dev, tdev, itfName, nil -} - -func CleanupWireGuardDevice(n *Nylon) error { - if n.Device != nil { - err := n.Device.Bind().Close() - if err != nil { - return err - } - n.Device.Close() + rt, err := factory.NewRuntime(n) + if err != nil { + return Runtime{}, err } - if n.wgUapi != nil { - err := n.wgUapi.Close() - if err != nil { - return err + // Tests use the WireGuard transport's UAPI without a socket. + if len(rt.Transports) != 0 { + if wg, ok := rt.Transports[0].(*adapter.Transport); ok { + n.wireGuard = &legacyWireGuard{transport: wg} } } - return nil + return rt, nil } diff --git a/core/sys_windows.go b/core/sys_windows.go index d9240230..4eab3c3f 100644 --- a/core/sys_windows.go +++ b/core/sys_windows.go @@ -1,14 +1,15 @@ package core import ( + "fmt" "log/slog" "net" "net/netip" "strconv" "strings" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/polyamide/transports/wireguard/ipc" - "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" "github.com/kmahyyg/go-network-compo/wintypes" ) @@ -42,8 +43,12 @@ func RemoveAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { return Exec(logger, "netsh", "interface", "ip", "delete", "address", ifName, addr.String()) } -func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { - ifId := wintypes.LUID((dev.(*tun.NativeTun)).LUID()) +func ConfigureRoute(logger *slog.Logger, dev polyamide.HostDevice, itfName string, route netip.Prefix) error { + native, ok := dev.(interface{ LUID() uint64 }) + if !ok { + return fmt.Errorf("host device does not expose a Windows LUID") + } + ifId := wintypes.LUID(native.LUID()) itf, err := ifId.Interface() if err != nil { return err @@ -60,8 +65,12 @@ func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route n } } -func RemoveRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { - ifId := wintypes.LUID((dev.(*tun.NativeTun)).LUID()) +func RemoveRoute(logger *slog.Logger, dev polyamide.HostDevice, itfName string, route netip.Prefix) error { + native, ok := dev.(interface{ LUID() uint64 }) + if !ok { + return fmt.Errorf("host device does not expose a Windows LUID") + } + ifId := wintypes.LUID(native.LUID()) itf, err := ifId.Interface() if err != nil { return err diff --git a/core/sys_wireguard.go b/core/sys_wireguard.go new file mode 100644 index 00000000..7e344361 --- /dev/null +++ b/core/sys_wireguard.go @@ -0,0 +1,86 @@ +package core + +import ( + "bufio" + "context" + "errors" + "log/slog" + "net" + "sync" + + "github.com/encodeous/nylon/log" + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/polyamide/transports/wireguard/adapter" + "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" + "github.com/encodeous/nylon/polyamide/transports/wireguard/device" +) + +// NewWireGuardTransport creates the WireGuard transport for n and serves Nylon IPC +// over its UAPI socket. A nil newBind uses the platform UDP bind. +func NewWireGuardTransport(n *Nylon, newBind func() conn.Bind) (*adapter.Transport, error) { + return adapter.New(adapter.Options{PrivateKey: polyamide.PrivateKey(n.Key), ListenPort: n.Port, NewBind: newBind, Logger: n.Log.With("module", log.ScopePolyamide), Debug: n.DBG_log_wireguard, + ControlHandlers: map[string]func(*bufio.ReadWriter) error{"get=nylon\n": func(w *bufio.ReadWriter) error { + err := HandleNylonIPC(n, w) + if errors.Is(err, ErrIPCStatusHandled) { + return device.ErrIPCStatusHandled + } + return err + }}, + }) +} + +// legacyWireGuard serves WireGuard's UAPI, which the wg tool and Nylon's IPC use. +// It exists for compatibility: only the platform runtimes create it, and it only +// covers the WireGuard transport. +type legacyWireGuard struct { + transport *adapter.Transport + listener net.Listener // nil when there is no UAPI socket + workers sync.WaitGroup +} + +// serve accepts UAPI connections until the listener closes. +func (w *legacyWireGuard) serve(ctx context.Context, logger *slog.Logger) { + if w.listener == nil { + return + } + w.workers.Add(1) + go func() { + defer w.workers.Done() + for ctx.Err() == nil { + c, err := w.listener.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return + } + logger.Debug("UAPI accept failed", "err", err) + continue + } + w.workers.Add(1) + go func() { + defer w.workers.Done() + defer c.Close() + stop := context.AfterFunc(ctx, func() { _ = c.Close() }) + defer stop() + w.transport.HandleUAPI(c) + }() + } + }() +} + +func (w *legacyWireGuard) close() error { + var err error + if w.listener != nil { + err = w.listener.Close() + } + w.workers.Wait() + return err +} + +// HandleUAPI serves one legacy UAPI connection with the WireGuard transport. +func (n *Nylon) HandleUAPI(c net.Conn) error { + if n.wireGuard == nil { + return polyamide.ErrUnsupported + } + n.wireGuard.transport.HandleUAPI(c) + return nil +} diff --git a/integration/apply_config_test.go b/integration/apply_config_test.go index ac48e13c..4081e953 100644 --- a/integration/apply_config_test.go +++ b/integration/apply_config_test.go @@ -9,9 +9,9 @@ import ( "time" "github.com/encodeous/nylon/core" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" "github.com/encodeous/nylon/state" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/goleak" ) @@ -185,15 +185,40 @@ func TestApplyCentralConfigRotatesPeerKeyWithoutChangingNextHop(t *testing.T) { waitForPayload(t, vh, errs, received, 1) newPrivateKey := state.GenerateKey() - b := vh.Nylons[vh.IndexOf("b")].Load() - assert.NoError(t, b.Device.SetPrivateKey(device.NoisePrivateKey(newPrivateKey))) - + bIdx := vh.IndexOf("b") + b := vh.Nylons[bIdx].Load() err, next := vh.Central.Clone() - assert.NoError(t, err) + require.NoError(t, err) next.Timestamp++ - next.Routers[vh.IndexOf("b")].PubKey = newPrivateKey.Pubkey() + next.Routers[bIdx].PubKey = newPrivateKey.Pubkey() + + // A local identity is fixed at construction. Restart the remote node while + // keeping the sender alive so its forwarding table must rebind the same hop. + b.Stop() + select { + case <-vh.nodeDone[bIdx]: + case <-time.After(5 * time.Second): + t.Fatal("timed out stopping remote node") + } + require.NoError(t, vh.Net.binds[bIdx].Close()) + local := vh.Local[bIdx] + local.Key = newPrivateKey + // Packet delivery holds linksMu while accessing bind slots. Pause delivery + // while the restarted node installs its replacement virtual bind and TUN. + vh.linksMu.Lock() + replacement, err := core.NewNylon(*next, local, *vh.LogLevel, "", map[string]any{"vnet": vh.Net}, state.NylonOptions{DBG_log_wireguard: true}, vh.Tunables) + vh.linksMu.Unlock() + require.NoError(t, err) + vh.Nylons[bIdx].Store(replacement) + done := make(chan struct{}) + vh.nodeDone[bIdx] = done + go func() { + defer close(done) + if err := replacement.Start(); err != nil { + errs <- err + } + }() - applyConfigAndWait(t, b, next) a := vh.Nylons[vh.IndexOf("a")].Load() applyConfigAndWait(t, a, next) @@ -329,5 +354,12 @@ func hasAppliedPeer(n *core.Nylon, id state.NodeId) bool { } func hasWGPeer(n *core.Nylon, key state.NyPublicKey) bool { - return n.Device.LookupPeer(device.NoisePublicKey(key)) != nil + for _, transport := range n.Transports { + for _, stat := range transport.Peers() { + if [32]byte(stat.Peer.PublicKey()) == [32]byte(key) { + return true + } + } + } + return false } diff --git a/integration/harness.go b/integration/harness.go index 84060199..e5773367 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -18,6 +18,7 @@ import ( "time" "github.com/encodeous/nylon/core" + "github.com/encodeous/nylon/polyamide" "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" "github.com/encodeous/nylon/polyamide/transports/wireguard/conn/bindtest" "github.com/encodeous/nylon/polyamide/transports/wireguard/device" @@ -108,6 +109,7 @@ type VirtualHarness struct { Local []state.LocalCfg Net *InMemoryNetwork Nylons []atomic.Pointer[core.Nylon] + nodeDone []chan struct{} Links []*VirtualLink linksMu sync.RWMutex Endpoints map[string]state.NodeId @@ -166,6 +168,7 @@ func (v *VirtualHarness) Start() chan error { v.Context = ctx v.Cancel = cancel v.Nylons = make([]atomic.Pointer[core.Nylon], len(v.Central.Routers)) + v.nodeDone = make([]chan struct{}, len(v.Central.Routers)) errChan := make(chan error, 128) // a large number so we dont get blocked vn := &InMemoryNetwork{} v.Net = vn @@ -206,7 +209,10 @@ func (v *VirtualHarness) Start() chan error { for idx, rt := range v.Central.Routers { sd := startDelay n := v.Nylons[idx].Load() + done := make(chan struct{}) + v.nodeDone[idx] = done go func() { + defer close(done) timer := time.NewTimer(sd) select { case <-timer.C: @@ -334,7 +340,22 @@ func (i *InMemoryNetwork) virtualInternet(pkt []byte, len int, from, to bindtest link.simulate(pkt, len, from, to, i) } -func (i *InMemoryNetwork) Bind(node state.NodeId) conn.Bind { +func (i *InMemoryNetwork) NewRuntime(n *core.Nylon) (core.Runtime, error) { + var host tun.Device + if n.NoTun { + host = tun.NewDummyDevice("nylon-vn") + } else { + host = i.Tun(n.LocalCfg.Id) + } + transport, err := core.NewWireGuardTransport(n, func() conn.Bind { return i.newWireGuardBind(n.LocalCfg.Id) }) + if err != nil { + _ = host.Close() + return core.Runtime{}, err + } + return core.Runtime{Transports: []polyamide.Transport{transport}, Host: host}, nil +} + +func (i *InMemoryNetwork) newWireGuardBind(node state.NodeId) conn.Bind { i.Lock() defer i.Unlock() epSendMapping := func(to bindtest.ChannelEndpoint2) bindtest.ChannelEndpoint2 { @@ -430,7 +451,7 @@ func (i *InMemoryNetwork) Send(node state.NodeId, src, dst string, pkt []byte, t numId := i.cfg.IndexOf(node) ipPkt := append(make([]byte, ipv4Size), pkt...) ip := ipPkt[0:ipv4Size] - ip[0] = 4 << 4 + ip[0] = 4<<4 | ipv4Size/4 binary.BigEndian.PutUint16(ip[2:4], uint16(ipv4Size+len(pkt))) ip[8] = ttl copy(ipPkt[12:16], netip.MustParseAddr(src).AsSlice()) diff --git a/integration/ipc_test.go b/integration/ipc_test.go index 7b991893..f3dce55f 100644 --- a/integration/ipc_test.go +++ b/integration/ipc_test.go @@ -12,7 +12,6 @@ import ( "time" "github.com/encodeous/nylon/core" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" "github.com/goccy/go-yaml" @@ -53,7 +52,7 @@ func ipcCall(t *testing.T, n *core.Nylon, req *protocol.IpcRequest) *protocol.Ip bufio.NewWriter(&outBuf), ) err = core.HandleNylonIPC(n, rw) - require.True(t, errors.Is(err, device.ErrIPCStatusHandled), "unexpected error: %v", err) + require.True(t, errors.Is(err, core.ErrIPCStatusHandled), "unexpected error: %v", err) resp := &protocol.IpcResponse{} um := protojson.UnmarshalOptions{DiscardUnknown: true} @@ -230,7 +229,7 @@ func TestIPCMalformedRequest(t *testing.T) { ) err := core.HandleNylonIPC(a, rw) // Should write an error response, not crash - assert.True(t, errors.Is(err, device.ErrIPCStatusHandled), "unexpected error: %v", err) + assert.True(t, errors.Is(err, core.ErrIPCStatusHandled), "unexpected error: %v", err) resp := &protocol.IpcResponse{} um := protojson.UnmarshalOptions{DiscardUnknown: true} assert.NoError(t, um.Unmarshal(outBuf.Bytes(), resp)) @@ -258,7 +257,7 @@ func TestIPCSocketResponseHasNoUAPIErrnoTrailer(t *testing.T) { a := vh.Nylons[vh.IndexOf("a")].Load() client, server := net.Pipe() defer client.Close() - go a.Device.IpcHandle(server) + go a.HandleUAPI(server) m := protojson.MarshalOptions{EmitUnpopulated: true} data, err := m.Marshal(&protocol.IpcRequest{ diff --git a/perf/vars.go b/perf/vars.go index 8114dab4..8455a5ac 100644 --- a/perf/vars.go +++ b/perf/vars.go @@ -8,27 +8,11 @@ import ( ) var ( - DispatchLatency = metric.NewHistogram("1m1s") - SendBatchSize = metric.NewHistogram("10s1s") - RecvBatchSize = metric.NewHistogram("10s1s") - SendsPerSecond = metric.NewCounter("10s1s") - RecvsPerSecond = metric.NewCounter("10s1s") - SentPacketPerSecond = metric.NewCounter("10s1s") - RecvPacketPerSecond = metric.NewCounter("10s1s") - SentBytesPerSecond = metric.NewCounter("10s1s") - RecvBytesPerSecond = metric.NewCounter("10s1s") + DispatchLatency = metric.NewHistogram("1m1s") ) func init() { http.Handle("/debug/metrics", metric.Handler(metric.Exposed)) - expvar.Publish("nylon:SendBatchSize", SendBatchSize) - expvar.Publish("nylon:RecvBatchSize", RecvBatchSize) - expvar.Publish("nylon:SentPacket/s", SentPacketPerSecond) - expvar.Publish("nylon:RecvPacket/s", RecvPacketPerSecond) - expvar.Publish("nylon:Sends/s", SendsPerSecond) - expvar.Publish("nylon:Recvs/s", RecvsPerSecond) - expvar.Publish("nylon:SentBytes/s", SentBytesPerSecond) - expvar.Publish("nylon:RecvBytes/s", RecvBytesPerSecond) expvar.Publish("nylon:DispatchLatency (µs)", DispatchLatency) } diff --git a/polyamide/README.md b/polyamide/README.md new file mode 100644 index 00000000..e89025c3 --- /dev/null +++ b/polyamide/README.md @@ -0,0 +1,18 @@ +# Polyamide + +Packet transports between Nylon peers. + +- The core owns TUN I/O, routing, and the filter chain. +- Transports own sockets, encryption, and peer state. +- Filters return decisions. Batch dispatch groups packets by transport. +- The WireGuard backend is in [adapter](transports/wireguard/adapter). + +Runtime: + +- `core.Runtime` supplies transports and a host device. +- `Runtime.LinksForNeighbour` supplies links across registered transport instances. +- Routes select a neighbour. Link selection chooses the peer and native endpoint. +- YAML configuration uses WireGuard. + +See [api.go](api.go) for the interfaces and [traffic_control.go](traffic_control.go) +for filtering and batch dispatch. diff --git a/polyamide/api.go b/polyamide/api.go new file mode 100644 index 00000000..f925806e --- /dev/null +++ b/polyamide/api.go @@ -0,0 +1,103 @@ +// Package polyamide defines the packet transport API. +package polyamide + +import ( + "context" + "errors" + "time" +) + +var ( + ErrClosed = errors.New("transport closed") + ErrStarted = errors.New("transport already started") + ErrNotStarted = errors.New("transport not started") + ErrInvalidKey = errors.New("invalid transport key") + ErrInvalidHandle = errors.New("foreign or retired transport handle") + ErrInvalidEndpoint = errors.New("invalid transport endpoint") + ErrMessageTooLarge = errors.New("control message too large") + ErrUnsupported = errors.New("unsupported transport feature") + ErrNoEndpoint = errors.New("peer has no configured endpoint") +) + +// Peers are compared by identity. Transport implementations must use pointers, and +// PreparePeer returns a new handle when a peer's public key changes. +type Peer interface { + Transport() Transport + ID() string + PublicKey() PublicKey +} + +// Endpoints are created by their transport. Nil selects the peer's configured endpoints. +type Endpoint interface { + Address() string +} + +const KeySize = 32 + +type PublicKey [KeySize]byte +type PrivateKey [KeySize]byte + +type PeerConfig struct { + ID string + PublicKey PublicKey + Passive bool +} + +// The transport owns Payload. Copy it to keep it after the callback returns. +type ControlMessage struct { + Payload []byte + From Peer + Endpoint Endpoint +} + +// Transports can call callbacks at the same time. Do not change peer configuration, +// call Start or Close, or wait for Nylon's dispatcher from a callback. +type Hooks struct { + RouteBatch TCFilter + Control func(ControlMessage) + DeliverHost func(packets [][]byte) error + // EndpointLearned reports a packet from a peer at an endpoint that the peer + // was not configured with, such as after the peer roams. + EndpointLearned func(peer Peer, endpoint Endpoint) +} + +// Byte counts include transport headers and reset when a peer handle is replaced. +type PeerStats struct { + Peer Peer + LastReceived time.Time + LastHandshake time.Time + TxBytes uint64 + RxBytes uint64 + PreferredEndpoint Endpoint + Keepalive time.Duration +} + +// The caller can reuse packet bytes after a send returns. +type Transport interface { + PrepareEndpoint(context.Context, string) (Endpoint, error) + // Cancelling the context stops the transport. Start does not wait for peers to connect. + Start(context.Context, Hooks) error + // The transport keeps replaced peer handles until RemovePeer. + PreparePeer(context.Context, PeerConfig) (Peer, error) + // SetEndpoints replaces the peer's addresses. The first is the default for packets + // sent without an endpoint. An empty list allows passive roaming. + SetEndpoints(context.Context, Peer, []Endpoint) error + RemovePeer(context.Context, Peer) error + // ControlMTU reports payload capacity, not discovered path MTU. + ControlMTU(Peer, Endpoint) (int, error) + SendControl(context.Context, Peer, Endpoint, []byte) error + SendPackets(context.Context, []OutboundPacket) error + // Peers returns statistics owned by the caller. Values may be read at different times. + Peers() []PeerStats + // SetHostMTU reports the host interface MTU. It can be called before Start. + SetHostMTU(mtu int) + // Close waits for transport workers and callbacks, even if startup failed. + Close() error +} + +type OutboundPacket struct { + Bytes []byte + To Peer + Endpoint Endpoint + Priority TCPriority +} diff --git a/polyamide/packet.go b/polyamide/packet.go new file mode 100644 index 00000000..08873c7c --- /dev/null +++ b/polyamide/packet.go @@ -0,0 +1,98 @@ +package polyamide + +import ( + "encoding/binary" + "net/netip" +) + +// IPBytes checks the declared IP length and returns a view without trailing +// transport padding. IPv4 options are accepted. IPv6 jumbograms are unsupported. +func IPBytes(data []byte) ([]byte, bool) { + if len(data) == 0 { + return nil, false + } + var length int + switch data[0] >> 4 { + case 4: + if len(data) < 20 { + return nil, false + } + header := int(data[0]&15) * 4 + length = int(binary.BigEndian.Uint16(data[2:4])) + if header < 20 || header > length { + return nil, false + } + case 6: + if len(data) < 40 { + return nil, false + } + length = 40 + int(binary.BigEndian.Uint16(data[4:6])) + default: + return nil, false + } + if length > len(data) { + return nil, false + } + return data[:length:length], true +} + +func (p TCElement) Incoming() bool { return p.From != nil } +func (p TCElement) Version() int { + if len(p.Bytes) == 0 { + return 0 + } + return int(p.Bytes[0] >> 4) +} +func (p TCElement) Source() netip.Addr { + if p.Version() == 4 && len(p.Bytes) >= 20 { + return netip.AddrFrom4([4]byte(p.Bytes[12:16])) + } + if p.Version() == 6 && len(p.Bytes) >= 40 { + return netip.AddrFrom16([16]byte(p.Bytes[8:24])) + } + return netip.Addr{} +} +func (p TCElement) Destination() netip.Addr { + if p.Version() == 4 && len(p.Bytes) >= 20 { + return netip.AddrFrom4([4]byte(p.Bytes[16:20])) + } + if p.Version() == 6 && len(p.Bytes) >= 40 { + return netip.AddrFrom16([16]byte(p.Bytes[24:40])) + } + return netip.Addr{} +} +func (p TCElement) TTL() uint8 { + if p.Version() == 4 && len(p.Bytes) >= 20 { + return p.Bytes[8] + } + if p.Version() == 6 && len(p.Bytes) >= 40 { + return p.Bytes[7] + } + return 0 +} + +// DecrementTTL modifies a borrowed IP packet in place. A zero TTL is left at +// zero. For IPv4, it updates the header checksum. +func (p TCElement) DecrementTTL() { + if p.TTL() == 0 { + return + } + if p.Version() == 6 { + p.Bytes[7]-- + return + } + p.Bytes[8]-- + checksum := uint32(binary.BigEndian.Uint16(p.Bytes[10:12])) + 0x100 + checksum += checksum >> 16 + binary.BigEndian.PutUint16(p.Bytes[10:12], uint16(checksum)) +} + +// HostDevice is owned by Nylon and is not managed by transports. +type HostDevice interface { + Read(bufs [][]byte, sizes []int, offset int) (int, error) + Write(bufs [][]byte, offset int) (int, error) + MTU() (int, error) + Name() (string, error) + BatchSize() int + Close() error +} diff --git a/polyamide/packet_test.go b/polyamide/packet_test.go new file mode 100644 index 00000000..9e1ee17e --- /dev/null +++ b/polyamide/packet_test.go @@ -0,0 +1,77 @@ +package polyamide + +import ( + "encoding/binary" + "testing" +) + +func checksum(data []byte) uint16 { + var sum uint32 + for i := 0; i < len(data); i += 2 { + sum += uint32(binary.BigEndian.Uint16(data[i : i+2])) + } + for sum>>16 != 0 { + sum = sum&65535 + sum>>16 + } + return ^uint16(sum) +} +func TestIPv4OptionsTTLAndPadding(t *testing.T) { + data := make([]byte, 32) + data[0], data[8], data[9] = 0x46, 2, 17 + binary.BigEndian.PutUint16(data[2:4], 28) + copy(data[12:20], []byte{10, 0, 0, 1, 10, 0, 0, 2}) + binary.BigEndian.PutUint16(data[10:12], checksum(data[:24])) + ip, ok := IPBytes(data) + if !ok || len(ip) != 28 || cap(ip) != 28 { + t.Fatal("incorrect packet extent") + } + p := TCElement{Bytes: ip} + if p.Source().String() != "10.0.0.1" || p.Destination().String() != "10.0.0.2" { + t.Fatal("incorrect addresses") + } + for want := uint8(1); ; want-- { + p.DecrementTTL() + if p.TTL() != want || checksum(ip[:24]) != 0 { + t.Fatal("invalid TTL or checksum") + } + if want == 0 { + break + } + } + p.DecrementTTL() + if p.TTL() != 0 || checksum(ip[:24]) != 0 { + t.Fatal("TTL underflow") + } +} +func TestMalformedIPAndIPv6(t *testing.T) { + for _, data := range [][]byte{nil, {0x45}, {0x80}, make([]byte, 20)} { + if _, ok := IPBytes(data); ok { + t.Fatal("accepted malformed packet") + } + } + ip := make([]byte, 20) + ip[0] = 0x46 + binary.BigEndian.PutUint16(ip[2:4], 20) + if _, ok := IPBytes(ip); ok { + t.Fatal("accepted options exceeding packet") + } + ip[0] = 0x45 + binary.BigEndian.PutUint16(ip[2:4], 21) + if _, ok := IPBytes(ip); ok { + t.Fatal("accepted truncated packet") + } + v6 := make([]byte, 48) + v6[0] = 0x60 + v6[7] = 1 + binary.BigEndian.PutUint16(v6[4:6], 4) + data, ok := IPBytes(v6) + if !ok || len(data) != 44 { + t.Fatal("invalid IPv6 extent") + } + p := TCElement{Bytes: data} + p.DecrementTTL() + p.DecrementTTL() + if p.TTL() != 0 { + t.Fatal("IPv6 TTL underflow") + } +} diff --git a/polyamide/traffic_control.go b/polyamide/traffic_control.go new file mode 100644 index 00000000..fbc83d85 --- /dev/null +++ b/polyamide/traffic_control.go @@ -0,0 +1,67 @@ +package polyamide + +import ( + "context" + "errors" +) + +type TCAction uint8 + +const ( + TcDrop TCAction = iota // zero decisions fail closed + TcBounce // deliver to the host writer + TcForward +) + +type TCPriority uint8 + +const ( + TcNormalPriority TCPriority = iota + TcHighPriority +) + +// The source owns Bytes. Filters can edit the bytes but must not resize or keep them. +type TCElement struct { + Bytes []byte + From Peer + Endpoint Endpoint +} + +type TCDecision struct { + Action TCAction + To Peer + Endpoint Endpoint + Priority TCPriority +} + +// TCFilter decides every packet in a batch. Each decision starts zeroed, so +// packets the filter leaves alone are dropped. +type TCFilter func(packets []TCElement, decisions []TCDecision) + +// DispatchBatch sends forwarded packets through their target transports, one +// batch per transport. Forward decisions are cleared, so only bounces remain. +func DispatchBatch(ctx context.Context, packets []TCElement, decisions []TCDecision) error { + if len(packets) == 0 { + return nil + } + batches := make(map[Transport][]OutboundPacket) + var result error + for i, packet := range packets { + decision := decisions[i] + if decision.Action != TcForward { + continue + } + if decision.To == nil || decision.To.Transport() == nil { + decisions[i] = TCDecision{} + result = errors.Join(result, ErrInvalidHandle) + continue + } + target := decision.To.Transport() + batches[target] = append(batches[target], OutboundPacket{Bytes: packet.Bytes, To: decision.To, Endpoint: decision.Endpoint, Priority: decision.Priority}) + decisions[i] = TCDecision{} + } + for transport, batch := range batches { + result = errors.Join(result, transport.SendPackets(ctx, batch)) + } + return result +} diff --git a/polyamide/traffic_control_test.go b/polyamide/traffic_control_test.go new file mode 100644 index 00000000..45ca1da6 --- /dev/null +++ b/polyamide/traffic_control_test.go @@ -0,0 +1,86 @@ +package polyamide + +import ( + "context" + "errors" + "github.com/stretchr/testify/require" + "slices" + "testing" +) + +type dispatchTransport struct { + Transport + batches [][]OutboundPacket + err error +} + +func (t *dispatchTransport) SendPackets(_ context.Context, packets []OutboundPacket) error { + batch := make([]OutboundPacket, len(packets)) + for i, p := range packets { + batch[i] = p + batch[i].Bytes = append([]byte(nil), p.Bytes...) + } + t.batches = append(t.batches, batch) + return t.err +} + +type dispatchPeer struct { + owner Transport + id string +} + +func (p *dispatchPeer) Transport() Transport { return p.owner } +func (p *dispatchPeer) ID() string { return p.id } +func (*dispatchPeer) PublicKey() PublicKey { return PublicKey{} } + +func TestDispatchBatchGroupsByTransport(t *testing.T) { + a, b := new(dispatchTransport), new(dispatchTransport) + pa1, pa2, pb := &dispatchPeer{a, "a1"}, &dispatchPeer{a, "a2"}, &dispatchPeer{b, "b"} + endpoint := dispatchEndpoint("127.0.0.1:1234") + expected := []TCDecision{ + {Action: TcForward, To: pa1, Endpoint: endpoint, Priority: TcHighPriority}, + {Action: TcForward, To: pb}, + {Action: TcForward, To: pa2}, + {Action: TcBounce}, + {Action: TcDrop}, + } + packets := make([]TCElement, len(expected)) + decisions := slices.Clone(expected) + for i := range packets { + packets[i].Bytes = []byte{byte(i)} + } + require.NoError(t, DispatchBatch(context.Background(), packets, decisions)) + require.Len(t, a.batches, 1) + require.Len(t, a.batches[0], 2) + require.Same(t, pa1, a.batches[0][0].To) + require.Same(t, pa2, a.batches[0][1].To) + require.Equal(t, endpoint, a.batches[0][0].Endpoint) + require.Equal(t, TcHighPriority, a.batches[0][0].Priority) + require.Equal(t, []byte{0}, a.batches[0][0].Bytes) + require.Equal(t, []byte{2}, a.batches[0][1].Bytes) + require.Len(t, b.batches, 1) + require.Len(t, b.batches[0], 1) + require.Same(t, pb, b.batches[0][0].To) + for _, d := range decisions[:3] { + require.Equal(t, TCDecision{}, d, "already submitted packets must not be forwarded again") + } + require.Equal(t, expected[3:], decisions[3:]) +} + +func TestDispatchBatchErrorsDoNotPreventOtherBatches(t *testing.T) { + failed := errors.New("send failed") + a, b := &dispatchTransport{err: failed}, new(dispatchTransport) + pa, pb := &dispatchPeer{a, "a"}, &dispatchPeer{b, "b"} + packets := []TCElement{{Bytes: []byte{1}}, {Bytes: []byte{2}}, {Bytes: []byte{3}}} + decisions := []TCDecision{{Action: TcForward, To: pa}, {Action: TcForward, To: pb}, {Action: TcForward}} + err := DispatchBatch(context.Background(), packets, decisions) + require.ErrorIs(t, err, failed) + require.ErrorIs(t, err, ErrInvalidHandle) + require.Len(t, a.batches, 1) + require.Len(t, b.batches, 1) + require.Equal(t, make([]TCDecision, 3), decisions) +} + +type dispatchEndpoint string + +func (e dispatchEndpoint) Address() string { return string(e) } diff --git a/polyamide/transports/wireguard/adapter/endpoints.go b/polyamide/transports/wireguard/adapter/endpoints.go new file mode 100644 index 00000000..4ab034a5 --- /dev/null +++ b/polyamide/transports/wireguard/adapter/endpoints.go @@ -0,0 +1,64 @@ +package adapter + +import ( + "context" + "fmt" + "net/netip" + + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" +) + +type endpoint struct { + owner *Transport + native conn.Endpoint +} + +func (e *endpoint) Address() string { return e.native.DstToString() } + +func (t *Transport) PrepareEndpoint(ctx context.Context, address string) (polyamide.Endpoint, error) { + if t.closed.Load() { + return nil, polyamide.ErrClosed + } + if err := ctx.Err(); err != nil { + return nil, err + } + ap, err := netip.ParseAddrPort(address) + if err != nil { + return nil, fmt.Errorf("%s: %w", address, polyamide.ErrInvalidEndpoint) + } + native, err := t.bind.ParseEndpoint(ap.String()) + if err != nil { + return nil, fmt.Errorf("parse endpoint: %w", err) + } + return &endpoint{owner: t, native: native}, nil +} + +func (t *Transport) resolveEndpoint(value polyamide.Endpoint) (conn.Endpoint, error) { + if value == nil { + return nil, nil + } + e, ok := value.(*endpoint) + if !ok || e == nil || e.owner != t || e.native == nil { + return nil, polyamide.ErrInvalidEndpoint + } + return e.native, nil +} + +func (t *Transport) rememberEndpoint(native conn.Endpoint) polyamide.Endpoint { + return &endpoint{owner: t, native: native} +} + +// WireGuard needs an initial peer address for the handshake. +func (t *Transport) ensureEndpoint(p *peer, endpoint conn.Endpoint) error { + p.endpointMu.Lock() + defer p.endpointMu.Unlock() + if len(p.raw.GetEndpoints()) != 0 { + return nil + } + if endpoint == nil { + return polyamide.ErrNoEndpoint + } + p.raw.SetEndpoints([]conn.Endpoint{endpoint}) + return nil +} diff --git a/polyamide/transports/wireguard/adapter/endpoints_test.go b/polyamide/transports/wireguard/adapter/endpoints_test.go new file mode 100644 index 00000000..7746abc2 --- /dev/null +++ b/polyamide/transports/wireguard/adapter/endpoints_test.go @@ -0,0 +1,38 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" + "github.com/stretchr/testify/require" +) + +func TestEndpointRetainsNativeStateAndRejectsForeignInstance(t *testing.T) { + transport := &Transport{bind: conn.NewDefaultBind(), configuredEndpoints: make(map[*peer][]conn.Endpoint)} + endpoint, err := transport.PrepareEndpoint(context.Background(), "127.0.0.1:1234") + require.NoError(t, err) + first, err := transport.resolveEndpoint(endpoint) + require.NoError(t, err) + cached, err := transport.resolveEndpoint(endpoint) + require.NoError(t, err) + require.Same(t, first, cached) + received, err := transport.bind.ParseEndpoint(endpoint.Address()) + require.NoError(t, err) + source := transport.rememberEndpoint(received) + cached, err = transport.resolveEndpoint(source) + require.NoError(t, err) + require.Same(t, received, cached) + other := &Transport{bind: conn.NewDefaultBind()} + _, err = other.resolveEndpoint(source) + require.ErrorIs(t, err, polyamide.ErrInvalidEndpoint) + _, err = other.resolveEndpoint(endpoint) + require.ErrorIs(t, err, polyamide.ErrInvalidEndpoint) + // Caller-held endpoints survive peer retirement and remain reusable. + p1, p2 := &peer{owner: transport}, &peer{owner: transport} + require.NoError(t, transport.SetEndpoints(context.Background(), p1, []polyamide.Endpoint{endpoint})) + p1.retired.Store(true) + require.NoError(t, transport.SetEndpoints(context.Background(), p2, []polyamide.Endpoint{endpoint})) + require.Same(t, first, transport.configuredEndpoints[p2][0]) +} diff --git a/polyamide/transports/wireguard/adapter/host_test.go b/polyamide/transports/wireguard/adapter/host_test.go new file mode 100644 index 00000000..891b96d9 --- /dev/null +++ b/polyamide/transports/wireguard/adapter/host_test.go @@ -0,0 +1,21 @@ +package adapter + +import ( + "testing" + + "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" + "github.com/stretchr/testify/require" +) + +func TestHostSinkReportsMTUChanges(t *testing.T) { + h := newHostSink(1) + h.setMTU(1280) + mtu, err := h.MTU() + require.NoError(t, err) + require.Equal(t, 1280, mtu) + require.Equal(t, tun.Event(tun.EventMTUUpdate), <-h.events) + h.setMTU(1280) + require.Empty(t, h.events) + require.NoError(t, h.Close()) + h.setMTU(1500) // must not send on the closed channel +} diff --git a/polyamide/transports/wireguard/adapter/packets.go b/polyamide/transports/wireguard/adapter/packets.go new file mode 100644 index 00000000..5742844d --- /dev/null +++ b/polyamide/transports/wireguard/adapter/packets.go @@ -0,0 +1,296 @@ +package adapter + +import ( + "cmp" + "context" + "encoding/binary" + "errors" + "fmt" + "os" + "slices" + "sync" + "sync/atomic" + + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" + "github.com/encodeous/nylon/polyamide/transports/wireguard/device" + "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" +) + +// Control messages use IP version 8, followed by a big-endian payload length. +const ( + controlVersion = 8 + controlHeaderSize = 3 + maxControlPayload = device.MaxContentSize - controlHeaderSize +) + +// controlPayload returns the payload of a control message. +func controlPayload(packet []byte) ([]byte, bool) { + if len(packet) < controlHeaderSize || packet[0]>>4 != controlVersion { + return nil, false + } + end := controlHeaderSize + int(binary.BigEndian.Uint16(packet[1:controlHeaderSize])) + if end > len(packet) { + return nil, false + } + return packet[controlHeaderSize:end:end], true +} + +type routeBatch struct { + packets []polyamide.TCElement + decisions []polyamide.TCDecision + bounced [][]byte // packets to deliver to the host +} + +// receive handles decrypted packets from raw. The device releases their buffers +// after it returns. +func (t *Transport) receive(raw *device.Peer, packets []device.InboundPacket) { + if t.closed.Load() || !t.started.Load() { + return + } + t.mu.RLock() + from := t.byRaw[raw] + t.mu.RUnlock() + if from == nil || from.retired.Load() { + return + } + hooks := t.hooks.Load() + b := t.batchPool.Get().(*routeBatch) + defer func() { + clear(b.packets) + clear(b.decisions) + clear(b.bounced) + b.packets, b.decisions, b.bounced = b.packets[:0], b.decisions[:0], b.bounced[:0] + t.batchPool.Put(b) + }() + for _, packet := range packets { + if packet.Endpoint == nil { + continue + } + source := t.rememberEndpoint(packet.Endpoint) + if payload, ok := controlPayload(packet.Packet); ok { + if hooks.Control != nil { + hooks.Control(polyamide.ControlMessage{Payload: payload, From: from, Endpoint: source}) + } + continue + } + if data, ok := polyamide.IPBytes(packet.Packet); ok { + b.packets = append(b.packets, polyamide.TCElement{Bytes: data, From: from, Endpoint: source}) + b.decisions = append(b.decisions, polyamide.TCDecision{}) + } + } + if len(b.packets) == 0 || hooks.RouteBatch == nil { + return + } + hooks.RouteBatch(b.packets, b.decisions) + if err := polyamide.DispatchBatch(t.runContext, b.packets, b.decisions); err != nil && t.runContext.Err() == nil && !t.closed.Load() { + t.opts.Logger.Warn("failed to dispatch packet batch", "err", err) + } + for i, decision := range b.decisions { + if decision.Action == polyamide.TcBounce { + b.bounced = append(b.bounced, b.packets[i].Bytes) + } + } + if len(b.bounced) != 0 && hooks.DeliverHost != nil { + if err := hooks.DeliverHost(b.bounced); err != nil && !t.closed.Load() { + t.opts.Logger.Warn("failed to deliver packets to host", "err", err) + } + } +} + +func (t *Transport) SendControl(ctx context.Context, value polyamide.Peer, selected polyamide.Endpoint, payload []byte) error { + if err := t.beginSubmission(); err != nil { + return err + } + defer t.submissions.Done() + p, err := t.checkPeer(value) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if !t.started.Load() { + return polyamide.ErrNotStarted + } + path, err := t.resolveEndpoint(selected) + if err != nil { + return err + } + if len(payload) > maxControlPayload { + return polyamide.ErrMessageTooLarge + } + if err := t.ensureEndpoint(p, path); err != nil { + return err + } + buffer := t.dev.GetMessageBuffer() + packet := buffer[device.MessageTransportHeaderSize:] + packet[0] = controlVersion << 4 + binary.BigEndian.PutUint16(packet[1:controlHeaderSize], uint16(len(payload))) + copy(packet[controlHeaderSize:], payload) + p.raw.SendPackets([]device.OutboundPacket{{Buffer: buffer, Size: controlHeaderSize + len(payload), Endpoint: path}}) + return nil +} + +func (t *Transport) SendPackets(ctx context.Context, packets []polyamide.OutboundPacket) error { + if err := t.beginSubmission(); err != nil { + return err + } + defer t.submissions.Done() + if t.closed.Load() { + return polyamide.ErrClosed + } + if err := ctx.Err(); err != nil { + return err + } + if !t.started.Load() { + return polyamide.ErrNotStarted + } + // Invalid packets are skipped so they cannot drop the rest of the batch. + var result error + submissions := make([]submission, 0, len(packets)) + for _, packet := range packets { + s, err := t.prepareSubmission(packet) + if err != nil { + result = errors.Join(result, err) + continue + } + submissions = append(submissions, s) + } + // Send high-priority packets first, then group by peer in batches the bind can send. + slices.SortStableFunc(submissions, func(a, b submission) int { return cmp.Compare(b.priority, a.priority) }) + byPeer := make(map[*peer][]device.OutboundPacket) + var order []*peer + for _, packet := range submissions { + if _, ok := byPeer[packet.peer]; !ok { + order = append(order, packet.peer) + } + buffer := t.dev.GetMessageBuffer() + copy(buffer[device.MessageTransportHeaderSize:], packet.data) + byPeer[packet.peer] = append(byPeer[packet.peer], device.OutboundPacket{Buffer: buffer, Size: len(packet.data), Endpoint: packet.endpoint}) + } + for _, p := range order { + for batch := range slices.Chunk(byPeer[p], t.dev.BatchSize()) { + p.raw.SendPackets(batch) + } + } + return result +} + +type submission struct { + data []byte + peer *peer + endpoint conn.Endpoint + priority polyamide.TCPriority +} + +// prepareSubmission validates packet and resolves its peer and endpoint. +func (t *Transport) prepareSubmission(packet polyamide.OutboundPacket) (submission, error) { + data, valid := polyamide.IPBytes(packet.Bytes) + if !valid { + return submission{}, fmt.Errorf("malformed outgoing IP packet") + } + if len(data) > device.MaxContentSize { + return submission{}, polyamide.ErrMessageTooLarge + } + if packet.Priority != polyamide.TcNormalPriority && packet.Priority != polyamide.TcHighPriority { + return submission{}, fmt.Errorf("invalid packet priority") + } + p, err := t.checkPeer(packet.To) + if err != nil { + return submission{}, err + } + endpoint, err := t.resolveEndpoint(packet.Endpoint) + if err != nil { + return submission{}, err + } + if err := t.ensureEndpoint(p, endpoint); err != nil { + return submission{}, err + } + return submission{data: data, peer: p, endpoint: endpoint, priority: packet.Priority}, nil +} + +func (t *Transport) beginSubmission() error { + t.submitMu.Lock() + defer t.submitMu.Unlock() + if t.closed.Load() { + return polyamide.ErrClosed + } + t.submissions.Add(1) + return nil +} + +// endpointLearned reports a new endpoint for raw once per address change. +func (t *Transport) endpointLearned(raw *device.Peer, native conn.Endpoint) { + hooks := t.hooks.Load() + if t.closed.Load() || hooks == nil || hooks.EndpointLearned == nil { + return + } + t.mu.RLock() + p := t.byRaw[raw] + t.mu.RUnlock() + if p == nil || p.retired.Load() { + return + } + destination := native.DstIPPort() + if last := p.learned.Swap(&destination); last != nil && *last == destination { + return + } + hooks.EndpointLearned(p, t.rememberEndpoint(native)) +} + +// SetHostMTU passes the host MTU to the engine through its TUN event reader. +func (t *Transport) SetHostMTU(mtu int) { t.host.setMTU(mtu) } + +// hostSink stands in for the engine's TUN. The transport delivers host packets +// through Hooks.DeliverHost, so the engine never reads or writes it. It only +// reports the host MTU, and never reports the interface going down. +type hostSink struct { + batchSize int + mtu atomic.Int32 + closed chan struct{} + events chan tun.Event + mu sync.Mutex // protects events against Close + done bool +} + +func newHostSink(batchSize int) *hostSink { + h := &hostSink{batchSize: batchSize, closed: make(chan struct{}), events: make(chan tun.Event, 1)} + h.mtu.Store(device.DefaultMTU) + return h +} + +func (h *hostSink) setMTU(mtu int) { + if h.mtu.Swap(int32(mtu)) == int32(mtu) { + return + } + h.mu.Lock() + defer h.mu.Unlock() + if h.done { + return + } + select { + case h.events <- tun.EventMTUUpdate: + default: // an update is already pending, and the reader loads the latest MTU + } +} +func (*hostSink) File() *os.File { return nil } +func (*hostSink) Name() (string, error) { return "polyamide-wireguard", nil } +func (h *hostSink) MTU() (int, error) { return int(h.mtu.Load()), nil } +func (h *hostSink) BatchSize() int { return h.batchSize } +func (h *hostSink) Events() <-chan tun.Event { return h.events } +func (h *hostSink) Read(_ [][]byte, _ []int, _ int) (int, error) { + <-h.closed + return 0, os.ErrClosed +} +func (*hostSink) Write(buffers [][]byte, _ int) (int, error) { return len(buffers), nil } +func (h *hostSink) Close() error { + h.mu.Lock() + defer h.mu.Unlock() + if !h.done { + h.done = true + close(h.closed) + close(h.events) + } + return nil +} diff --git a/polyamide/transports/wireguard/adapter/transport.go b/polyamide/transports/wireguard/adapter/transport.go new file mode 100644 index 00000000..769501a0 --- /dev/null +++ b/polyamide/transports/wireguard/adapter/transport.go @@ -0,0 +1,400 @@ +// Package adapter implements Polyamide using wireguard-go. +package adapter + +import ( + "bufio" + "bytes" + "cmp" + "context" + "crypto/ecdh" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "net" + "net/netip" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" + "github.com/encodeous/nylon/polyamide/transports/wireguard/device" +) + +type Options struct { + PrivateKey polyamide.PrivateKey + ListenPort uint16 + // Nil uses the platform UDP bind. + NewBind func() conn.Bind + Logger *slog.Logger + Debug bool + ControlHandlers map[string]func(*bufio.ReadWriter) error +} + +type Transport struct { + runContext context.Context + opts Options + bind conn.Bind + localPublic device.NoisePublicKey + // ops protects configuration and device startup and shutdown. Callbacks do not lock it. + ops sync.Mutex + mu sync.RWMutex + dev *device.Device + peers map[device.NoisePublicKey]*peer + byRaw map[*device.Peer]*peer + configuredEndpoints map[*peer][]conn.Endpoint + started atomic.Bool + closed atomic.Bool + hooks atomic.Pointer[polyamide.Hooks] + // Stop accepting packets before waiting for workers. Callbacks can send replies + // without taking the configuration lock. + submitMu sync.Mutex + submissions sync.WaitGroup + done chan struct{} + stopContext func() bool + batchPool sync.Pool + host *hostSink +} + +type peer struct { + owner *Transport + id string + public device.NoisePublicKey + passive bool // protected by ops + raw *device.Peer // published under mu before startup. It does not change after startup. + retired atomic.Bool + endpointMu sync.Mutex + // learned is the last endpoint reported through Hooks.EndpointLearned. + learned atomic.Pointer[netip.AddrPort] +} + +func (p *peer) Transport() polyamide.Transport { return p.owner } +func (p *peer) ID() string { return p.id } +func (p *peer) PublicKey() polyamide.PublicKey { return polyamide.PublicKey(p.public) } + +var _ polyamide.Transport = (*Transport)(nil) + +func New(opts Options) (*Transport, error) { + if opts.PrivateKey == (polyamide.PrivateKey{}) { + return nil, fmt.Errorf("local private key: %w", polyamide.ErrInvalidKey) + } + local, err := ecdh.X25519().NewPrivateKey(opts.PrivateKey[:]) + if err != nil { + return nil, fmt.Errorf("local private key: %w", polyamide.ErrInvalidKey) + } + newBind := opts.NewBind + if newBind == nil { + newBind = conn.NewDefaultBind + } + bind := newBind() + if bind == nil { + return nil, errors.New("WireGuard bind factory returned nil") + } + if bind.BatchSize() <= 0 { + _ = bind.Close() + return nil, errors.New("WireGuard bind must have a positive batch size") + } + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.ControlHandlers != nil { + handlers := make(map[string]func(*bufio.ReadWriter) error, len(opts.ControlHandlers)) + for k, v := range opts.ControlHandlers { + handlers[k] = v + } + opts.ControlHandlers = handlers + } + t := &Transport{opts: opts, bind: bind, localPublic: device.NoisePublicKey(local.PublicKey().Bytes()), peers: make(map[device.NoisePublicKey]*peer), + byRaw: make(map[*device.Peer]*peer), configuredEndpoints: make(map[*peer][]conn.Endpoint), done: make(chan struct{})} + t.batchPool.New = func() any { return new(routeBatch) } + t.host = newHostSink(bind.BatchSize()) + return t, nil +} + +func (t *Transport) Start(ctx context.Context, hooks polyamide.Hooks) (err error) { + t.ops.Lock() + defer func() { + t.ops.Unlock() + if err != nil && !errors.Is(err, polyamide.ErrStarted) { + _ = t.Close() + } + }() + if t.closed.Load() { + return polyamide.ErrClosed + } + if t.started.Load() { + return polyamide.ErrStarted + } + if err = ctx.Err(); err != nil { + return err + } + t.runContext = ctx + t.hooks.Store(&hooks) + logger := &device.Logger{ + Verbosef: func(format string, args ...any) { + if t.opts.Debug { + t.opts.Logger.Debug(fmt.Sprintf(format, args...)) + } + }, + Errorf: func(format string, args ...any) { t.opts.Logger.Error(fmt.Sprintf(format, args...)) }, + } + dev := device.NewDeviceWithHooks(t.host, t.bind, logger, device.Hooks{Receive: t.receive, EndpointLearned: t.endpointLearned}) + t.mu.Lock() + t.dev = dev + t.mu.Unlock() + if err = dev.IpcSet(fmt.Sprintf("private_key=%s\nlisten_port=%d\n", hex.EncodeToString(t.opts.PrivateKey[:]), t.opts.ListenPort)); err != nil { + return err + } + for _, p := range t.peers { + if err = t.attachPeer(dev, p); err != nil { + return err + } + } + if err = ctx.Err(); err != nil { + return err + } + if err = dev.Up(); err != nil { + return err + } + for op, handler := range t.opts.ControlHandlers { + dev.IpcHandler[op] = handler + } + if t.closed.Load() { + return polyamide.ErrClosed + } + t.started.Store(true) + t.stopContext = context.AfterFunc(ctx, func() { _ = t.Close() }) + return nil +} + +func (t *Transport) attachPeer(dev *device.Device, p *peer) error { + raw, err := dev.NewPeer(p.public) + if err != nil { + return err + } + raw.SetPreferRoaming(p.passive) + raw.SetEndpoints(slices.Clone(t.configuredEndpoints[p])) + t.mu.Lock() + p.raw = raw + t.byRaw[raw] = p + t.mu.Unlock() + if t.started.Load() { + raw.Start() + } + return nil +} + +func (t *Transport) PreparePeer(ctx context.Context, cfg polyamide.PeerConfig) (polyamide.Peer, error) { + t.ops.Lock() + defer t.ops.Unlock() + if t.closed.Load() { + return nil, polyamide.ErrClosed + } + if err := ctx.Err(); err != nil { + return nil, err + } + if cfg.ID == "" { + return nil, polyamide.ErrInvalidKey + } + public := device.NoisePublicKey(cfg.PublicKey) + if public == (device.NoisePublicKey{}) { + return nil, fmt.Errorf("peer key: %w", polyamide.ErrInvalidKey) + } + if public == t.localPublic { + return nil, fmt.Errorf("peer is local identity: %w", polyamide.ErrInvalidKey) + } + if p := t.peers[public]; p != nil { + if p.id != cfg.ID { + return nil, fmt.Errorf("public key is bound to another ID: %w", polyamide.ErrInvalidKey) + } + p.passive = cfg.Passive + if p.raw != nil { + p.raw.SetPreferRoaming(cfg.Passive) + } + return p, nil + } + p := &peer{owner: t, id: cfg.ID, public: public, passive: cfg.Passive} + if t.dev != nil { + if err := t.attachPeer(t.dev, p); err != nil { + return nil, err + } + } + t.mu.Lock() + t.peers[public] = p + t.mu.Unlock() + return p, nil +} + +func (t *Transport) checkPeer(value polyamide.Peer) (*peer, error) { + if t.closed.Load() { + return nil, polyamide.ErrClosed + } + p, ok := value.(*peer) + if !ok || p == nil || p.owner != t || p.retired.Load() { + return nil, polyamide.ErrInvalidHandle + } + return p, nil +} + +func (t *Transport) SetEndpoints(ctx context.Context, value polyamide.Peer, locators []polyamide.Endpoint) error { + t.ops.Lock() + defer t.ops.Unlock() + p, err := t.checkPeer(value) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + endpoints := make([]conn.Endpoint, len(locators)) + for i, value := range locators { + endpoint, err := t.resolveEndpoint(value) + if err != nil { + return err + } + if endpoint == nil { + return polyamide.ErrInvalidEndpoint + } + endpoints[i] = endpoint + } + t.mu.Lock() + t.configuredEndpoints[p] = endpoints + t.mu.Unlock() + if p.raw != nil { + p.endpointMu.Lock() + p.raw.SetEndpoints(slices.Clone(endpoints)) + p.endpointMu.Unlock() + } + return nil +} + +func (t *Transport) RemovePeer(ctx context.Context, value polyamide.Peer) error { + t.ops.Lock() + defer t.ops.Unlock() + if t.closed.Load() { + return polyamide.ErrClosed + } + if err := ctx.Err(); err != nil { + return err + } + p, ok := value.(*peer) + if !ok || p == nil || p.owner != t { + return polyamide.ErrInvalidHandle + } + if p.retired.Swap(true) { + return nil + } + t.mu.Lock() + delete(t.peers, p.public) + delete(t.byRaw, p.raw) + delete(t.configuredEndpoints, p) + t.mu.Unlock() + // Callbacks also lock mu, so unlock it before waiting for the peer to stop. + if t.dev != nil { + t.dev.RemovePeer(p.public) + } + return nil +} + +func (t *Transport) ControlMTU(value polyamide.Peer, selected polyamide.Endpoint) (int, error) { + _, err := t.checkPeer(value) + if err != nil { + return 0, err + } + if _, err := t.resolveEndpoint(selected); err != nil { + return 0, err + } + return maxControlPayload, nil +} + +// ListenPort reports the bound UDP port, or zero before Start. +func (t *Transport) ListenPort() uint16 { + t.mu.RLock() + dev := t.dev + t.mu.RUnlock() + if dev == nil || !t.started.Load() { + return 0 + } + return dev.ListenPort() +} + +func (t *Transport) Peers() []polyamide.PeerStats { + t.mu.RLock() + peers := make([]*peer, 0, len(t.peers)) + for _, p := range t.peers { + peers = append(peers, p) + } + t.mu.RUnlock() + // A replaced peer shares its ID with the replacement until it is removed. + slices.SortFunc(peers, func(a, b *peer) int { + return cmp.Or(cmp.Compare(a.id, b.id), bytes.Compare(a.public[:], b.public[:])) + }) + stats := make([]polyamide.PeerStats, 0, len(peers)) + for _, p := range peers { + stat := polyamide.PeerStats{Peer: p} + t.mu.RLock() + raw := p.raw + t.mu.RUnlock() + if raw != nil { + s := raw.Status() + stat.LastReceived = raw.LastReceivedPacket() + if stat.LastReceived.UnixNano() == 0 { + stat.LastReceived = time.Time{} + } + stat.LastHandshake = s.LatestHandshakeTime() + stat.TxBytes, stat.RxBytes = s.TxBytes, s.RxBytes + if s.Endpoint != "" { + stat.PreferredEndpoint, _ = t.PrepareEndpoint(context.Background(), s.Endpoint) + } + stat.Keepalive = time.Duration(s.PersistentKeepaliveInterval) * time.Second + } + stats = append(stats, stat) + } + return stats +} + +func (t *Transport) Close() error { + if !t.closed.CompareAndSwap(false, true) { + <-t.done + return nil + } + t.ops.Lock() + defer t.ops.Unlock() + if t.stopContext != nil { + t.stopContext() + } + // Stop accepting submissions before waiting for workers. Replies attempted + // during shutdown return ErrClosed. + t.submitMu.Lock() + t.submitMu.Unlock() + t.submissions.Wait() + if t.dev != nil { + t.dev.Close() + } else { + _ = t.bind.Close() + } + t.mu.Lock() + for _, p := range t.peers { + p.retired.Store(true) + } + clear(t.peers) + clear(t.byRaw) + clear(t.configuredEndpoints) + t.mu.Unlock() + close(t.done) + return nil +} + +func (t *Transport) HandleUAPI(c net.Conn) { + if !t.started.Load() || t.closed.Load() { + _ = c.Close() + return + } + t.mu.RLock() + dev := t.dev + t.mu.RUnlock() + // Peers are managed through the transport, so UAPI cannot change them. + dev.IpcHandleReadOnly(c) +} diff --git a/polyamide/transports/wireguard/adapter/transport_test.go b/polyamide/transports/wireguard/adapter/transport_test.go new file mode 100644 index 00000000..9adcbcf6 --- /dev/null +++ b/polyamide/transports/wireguard/adapter/transport_test.go @@ -0,0 +1,417 @@ +package adapter_test + +import ( + "bufio" + "bytes" + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/netip" + "strings" + "sync" + "testing" + "time" + + "github.com/encodeous/nylon/polyamide" + "github.com/encodeous/nylon/polyamide/transports/wireguard/adapter" + "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" + "github.com/encodeous/nylon/polyamide/transports/wireguard/tun/tuntest" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" +) + +func privateKey(t *testing.T) [32]byte { + t.Helper() + k, err := ecdh.X25519().GenerateKey(rand.Reader) + require.NoError(t, err) + return [32]byte(k.Bytes()) +} + +func publicKey(t *testing.T, key [32]byte) polyamide.PublicKey { + t.Helper() + k, err := ecdh.X25519().NewPrivateKey(key[:]) + require.NoError(t, err) + return polyamide.PublicKey(k.PublicKey().Bytes()) +} + +func newTransport(t *testing.T, key [32]byte) *adapter.Transport { + t.Helper() + wg, err := adapter.New(adapter.Options{PrivateKey: key, Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, wg.Close()) }) + return wg +} + +type pair struct { + transports [2]*adapter.Transport + peers [2]polyamide.Peer + paths [2]polyamide.Endpoint +} + +func newPair(t *testing.T, hooks [2]polyamide.Hooks) pair { + t.Helper() + ctx := context.Background() + keys := [2][32]byte{privateKey(t), privateKey(t)} + var p pair + for i := range p.transports { + p.transports[i] = newTransport(t, keys[i]) + peer, err := p.transports[i].PreparePeer(ctx, polyamide.PeerConfig{ID: []string{"b", "a"}[i], PublicKey: publicKey(t, keys[i^1])}) + require.NoError(t, err) + p.peers[i] = peer + } + for i := range p.transports { + require.NoError(t, p.transports[i].Start(ctx, hooks[i])) + } + for i := range p.transports { + port := p.transports[i^1].ListenPort() + endpoint, err := p.transports[i].PrepareEndpoint(ctx, netip.AddrPortFrom(netip.MustParseAddr("127.0.0.1"), port).String()) + require.NoError(t, err) + p.paths[i] = endpoint + require.NoError(t, p.transports[i].SetEndpoints(ctx, p.peers[i], []polyamide.Endpoint{endpoint})) + } + return p +} + +func receive[T any](t *testing.T, ch <-chan T) T { + t.Helper() + select { + case value := <-ch: + return value + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for transport delivery") + var zero T + return zero + } +} + +func TestEncryptedControlAndHostPackets(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx := context.Background() + type message struct { + payload []byte + peer polyamide.Peer + path polyamide.Endpoint + } + control := make(chan message, 4) + host := make(chan []byte, 4) + replyErrors := make(chan error, 1) + var p pair + var hooks [2]polyamide.Hooks + hooks[0].Control = func(m polyamide.ControlMessage) { control <- message{bytes.Clone(m.Payload), m.From, m.Endpoint} } + hooks[1].Control = func(m polyamide.ControlMessage) { + control <- message{bytes.Clone(m.Payload), m.From, m.Endpoint} + replyErrors <- p.transports[1].SendControl(ctx, m.From, m.Endpoint, []byte("reply")) + } + for i := range hooks { + hooks[i].RouteBatch = func(packets []polyamide.TCElement, decisions []polyamide.TCDecision) { + for j, packet := range packets { + if packet.Incoming() { + packet.DecrementTTL() + decisions[j].Action = polyamide.TcBounce + } else { + decisions[j] = polyamide.TCDecision{Action: polyamide.TcForward, To: p.peers[i]} + } + } + } + hooks[i].DeliverHost = func(packets [][]byte) error { + for _, packet := range packets { + host <- bytes.Clone(packet) + } + return nil + } + } + p = newPair(t, hooks) + payload := []byte("request") + require.NoError(t, p.transports[0].SendControl(ctx, p.peers[0], p.paths[0], payload)) + copy(payload, []byte("changed")) // caller storage is reusable immediately + request := receive(t, control) + require.Equal(t, "request", string(request.payload)) + require.Same(t, p.peers[1], request.peer) + require.Equal(t, p.paths[1], request.path) + require.NoError(t, receive(t, replyErrors)) + reply := receive(t, control) + require.Equal(t, "reply", string(reply.payload)) + require.Same(t, p.peers[0], reply.peer) + + packet := tuntest.Ping(netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.1")) + expected := bytes.Clone(packet) + polyamide.TCElement{Bytes: expected}.DecrementTTL() + require.NoError(t, p.transports[0].SendPackets(ctx, []polyamide.OutboundPacket{{Bytes: packet, To: p.peers[0]}})) + clear(packet) + require.Equal(t, expected, receive(t, host)) + stat := p.transports[1].Peers()[0] + require.Greater(t, stat.RxBytes, uint64(0)) + require.False(t, stat.LastHandshake.IsZero()) + require.False(t, stat.LastReceived.IsZero()) +} + +func TestPeerReplacementAndHandleValidation(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx := context.Background() + wg := newTransport(t, privateKey(t)) + cfg := polyamide.PeerConfig{ID: "remote", PublicKey: publicKey(t, privateKey(t))} + old, err := wg.PreparePeer(ctx, cfg) + require.NoError(t, err) + oldPath, err := wg.PrepareEndpoint(ctx, "127.0.0.1:12345") + require.NoError(t, err) + require.NoError(t, wg.SetEndpoints(ctx, old, []polyamide.Endpoint{oldPath})) + clear(cfg.PublicKey[:]) + require.NotEqual(t, cfg.PublicKey, old.PublicKey()) + copyOfKey := old.PublicKey() + clear(copyOfKey[:]) + require.NotEqual(t, copyOfKey, old.PublicKey()) + unchanged, err := wg.PreparePeer(ctx, polyamide.PeerConfig{ID: "remote", PublicKey: old.PublicKey()}) + require.NoError(t, err) + require.Same(t, old, unchanged) + require.NoError(t, wg.Start(ctx, polyamide.Hooks{})) + + replacement, err := wg.PreparePeer(ctx, polyamide.PeerConfig{ID: "remote", PublicKey: publicKey(t, privateKey(t))}) + require.NoError(t, err) + require.Equal(t, old.ID(), replacement.ID()) + require.NotSame(t, old, replacement) + require.Len(t, wg.Peers(), 2) + require.NoError(t, wg.SetEndpoints(ctx, replacement, []polyamide.Endpoint{oldPath})) + other := newTransport(t, privateKey(t)) + require.ErrorIs(t, other.SetEndpoints(ctx, old, nil), polyamide.ErrInvalidHandle) + require.NoError(t, wg.RemovePeer(ctx, old)) + require.NoError(t, wg.RemovePeer(ctx, old)) + require.Len(t, wg.Peers(), 1) + require.ErrorIs(t, wg.SendControl(ctx, old, oldPath, nil), polyamide.ErrInvalidHandle) + err = wg.SetEndpoints(ctx, old, []polyamide.Endpoint{oldPath}) + require.ErrorIs(t, err, polyamide.ErrInvalidHandle) + recreated, err := wg.PreparePeer(ctx, polyamide.PeerConfig{ID: "remote", PublicKey: old.PublicKey()}) + require.NoError(t, err) + require.NotSame(t, old, recreated) +} + +func TestControlLimitsAndUnsupportedInputs(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx := context.Background() + wg := newTransport(t, privateKey(t)) + for _, cfg := range []polyamide.PeerConfig{ + {ID: "bad", PublicKey: polyamide.PublicKey{}}, + {PublicKey: publicKey(t, privateKey(t))}, + } { + _, err := wg.PreparePeer(ctx, cfg) + require.ErrorIs(t, err, polyamide.ErrInvalidKey) + } + + p, err := wg.PreparePeer(ctx, polyamide.PeerConfig{ID: "peer", PublicKey: publicKey(t, privateKey(t))}) + require.NoError(t, err) + require.ErrorIs(t, wg.SendControl(ctx, p, nil, nil), polyamide.ErrNotStarted) + _, err = wg.PrepareEndpoint(ctx, "example.invalid:12345") + require.ErrorIs(t, err, polyamide.ErrInvalidEndpoint) + path, err := wg.PrepareEndpoint(ctx, "[::1]:12345") + require.NoError(t, err) + require.NoError(t, wg.Start(ctx, polyamide.Hooks{})) + require.ErrorIs(t, wg.Start(ctx, polyamide.Hooks{}), polyamide.ErrStarted) + require.ErrorIs(t, wg.SendControl(ctx, p, nil, nil), polyamide.ErrNoEndpoint) + mtu, err := wg.ControlMTU(p, path) + require.NoError(t, err) + require.ErrorIs(t, wg.SendControl(ctx, p, path, make([]byte, mtu+1)), polyamide.ErrMessageTooLarge) + require.NoError(t, wg.SetEndpoints(ctx, p, []polyamide.Endpoint{path})) + require.NoError(t, wg.SendControl(ctx, p, path, make([]byte, mtu))) + require.NoError(t, wg.Close()) + require.NoError(t, wg.Close()) + require.ErrorIs(t, wg.SendPackets(ctx, nil), polyamide.ErrClosed) +} + +func TestCloseJoinsCallbackWithImmediateReply(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + entered, release := make(chan struct{}), make(chan struct{}) + errors := make(chan error, 1) + var p pair + var once sync.Once + hooks := [2]polyamide.Hooks{{}, {Control: func(m polyamide.ControlMessage) { + once.Do(func() { close(entered) }) + <-release + errors <- p.transports[1].SendControl(context.Background(), m.From, m.Endpoint, []byte("reply")) + }}} + p = newPair(t, hooks) + require.NoError(t, p.transports[0].SendControl(context.Background(), p.peers[0], p.paths[0], []byte("request"))) + receive(t, entered) + closed := make(chan struct{}) + go func() { _ = p.transports[1].Close(); close(closed) }() + require.Eventually(t, func() bool { return errorsIsClosed(p.transports[1].SendPackets(context.Background(), nil)) }, time.Second, time.Millisecond) + select { + case <-closed: + t.Fatal("Close returned before its callback finished") + default: + } + close(release) + require.ErrorIs(t, receive(t, errors), polyamide.ErrClosed) + receive(t, closed) +} + +func errorsIsClosed(err error) bool { return errors.Is(err, polyamide.ErrClosed) } + +type failingBind struct { + conn.Bind + failure error +} + +func (b failingBind) Open(uint16) ([]conn.ReceiveFunc, uint16, error) { return nil, 0, b.failure } + +func TestStartFailureAndContextLifetime(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + failure := errors.New("bind failure") + wg, err := adapter.New(adapter.Options{PrivateKey: privateKey(t), NewBind: func() conn.Bind { return failingBind{Bind: conn.NewDefaultBind(), failure: failure} }, Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}) + require.NoError(t, err) + require.ErrorIs(t, wg.Start(context.Background(), polyamide.Hooks{}), failure) + require.NoError(t, wg.Close()) + require.ErrorIs(t, wg.SendPackets(context.Background(), nil), polyamide.ErrClosed) + + wg = newTransport(t, privateKey(t)) + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, wg.Start(ctx, polyamide.Hooks{})) + cancel() + require.Eventually(t, func() bool { return errors.Is(wg.SendPackets(context.Background(), nil), polyamide.ErrClosed) }, time.Second, time.Millisecond) + require.NoError(t, wg.Close()) +} + +type trackedBind struct { + conn.Bind + opened int + closed int + batchSize int +} + +func (b *trackedBind) BatchSize() int { return b.batchSize } +func (b *trackedBind) Open(port uint16) ([]conn.ReceiveFunc, uint16, error) { + b.opened++ + return b.Bind.Open(port) +} +func (b *trackedBind) Close() error { b.closed++; return b.Bind.Close() } + +func TestTransportCreatesAndOwnsBind(t *testing.T) { + bind := &trackedBind{Bind: conn.NewDefaultBind(), batchSize: 1} + created := 0 + wg, err := adapter.New(adapter.Options{PrivateKey: privateKey(t), NewBind: func() conn.Bind { created++; return bind }}) + require.NoError(t, err) + require.Equal(t, 1, created) + require.Zero(t, bind.opened, "construction must not listen") + require.NoError(t, wg.Close()) + require.NoError(t, wg.Close()) + require.Equal(t, 1, bind.closed, "unstarted bind is owned and closed once") + invalid := &trackedBind{Bind: conn.NewDefaultBind()} + _, err = adapter.New(adapter.Options{PrivateKey: privateKey(t), NewBind: func() conn.Bind { return invalid }}) + require.Error(t, err) + require.Equal(t, 1, invalid.closed, "failed construction must release its bind") + _, err = adapter.New(adapter.Options{PrivateKey: privateKey(t), NewBind: func() conn.Bind { return nil }}) + require.Error(t, err) +} + +func TestExplicitEndpointStartsPeerAndReplyUsesReceivedValue(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx := context.Background() + received := make(chan []byte, 2) + replies := make(chan error, 1) + var p pair + hooks := [2]polyamide.Hooks{ + {Control: func(m polyamide.ControlMessage) { received <- bytes.Clone(m.Payload) }}, + {Control: func(m polyamide.ControlMessage) { + replies <- p.transports[1].SendControl(ctx, m.From, m.Endpoint, []byte("reply")) + }}, + } + p = newPair(t, hooks) + for i := range p.transports { + require.NoError(t, p.transports[i].SetEndpoints(ctx, p.peers[i], nil)) + } + require.NoError(t, p.transports[0].SendControl(ctx, p.peers[0], p.paths[0], []byte("request"))) + require.NoError(t, receive(t, replies)) + require.Equal(t, []byte("reply"), receive(t, received)) + // Endpoints can be reused by peer generations on the same transport. + endpoint := p.paths[0] + require.NoError(t, p.transports[0].SetEndpoints(ctx, p.peers[0], []polyamide.Endpoint{endpoint})) + require.ErrorIs(t, p.transports[0].SetEndpoints(ctx, p.peers[0], []polyamide.Endpoint{nil}), polyamide.ErrInvalidEndpoint) +} + +func TestSendPacketsSkipsInvalidPackets(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx := context.Background() + host := make(chan []byte, 4) + var hooks [2]polyamide.Hooks + hooks[1].RouteBatch = func(packets []polyamide.TCElement, decisions []polyamide.TCDecision) { + for i := range packets { + decisions[i].Action = polyamide.TcBounce + } + } + hooks[1].DeliverHost = func(packets [][]byte) error { + for _, packet := range packets { + host <- bytes.Clone(packet) + } + return nil + } + p := newPair(t, hooks) + // A peer without an endpoint cannot be sent to, but it must not drop the rest of the batch. + unreachable, err := p.transports[0].PreparePeer(ctx, polyamide.PeerConfig{ID: "unreachable", PublicKey: publicKey(t, privateKey(t))}) + require.NoError(t, err) + packet := tuntest.Ping(netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.1")) + err = p.transports[0].SendPackets(ctx, []polyamide.OutboundPacket{{Bytes: packet, To: unreachable}, {Bytes: packet, To: p.peers[0]}}) + require.ErrorIs(t, err, polyamide.ErrNoEndpoint) + require.Equal(t, packet, receive(t, host)) +} + +func TestEndpointLearnedReportsUnconfiguredSourceOnce(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx := context.Background() + learned := make(chan polyamide.Endpoint, 8) + control := make(chan struct{}, 8) + var hooks [2]polyamide.Hooks + hooks[1].EndpointLearned = func(_ polyamide.Peer, endpoint polyamide.Endpoint) { learned <- endpoint } + hooks[1].Control = func(polyamide.ControlMessage) { control <- struct{}{} } + p := newPair(t, hooks) + // Transport 1 only knows a stale address for transport 0. + stale, err := p.transports[1].PrepareEndpoint(ctx, "127.0.0.1:1") + require.NoError(t, err) + require.NoError(t, p.transports[1].SetEndpoints(ctx, p.peers[1], []polyamide.Endpoint{stale})) + for range 3 { + require.NoError(t, p.transports[0].SendControl(ctx, p.peers[0], p.paths[0], []byte("hello"))) + receive(t, control) + } + endpoint := receive(t, learned) + require.Equal(t, netip.AddrPortFrom(netip.MustParseAddr("127.0.0.1"), p.transports[0].ListenPort()).String(), endpoint.Address()) + require.Empty(t, learned) +} + +func TestUAPIIsReadOnly(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx := context.Background() + wg := newTransport(t, privateKey(t)) + remote := publicKey(t, privateKey(t)) + peer, err := wg.PreparePeer(ctx, polyamide.PeerConfig{ID: "remote", PublicKey: remote}) + require.NoError(t, err) + require.NoError(t, wg.Start(ctx, polyamide.Hooks{})) + client, server := net.Pipe() + defer client.Close() + go wg.HandleUAPI(server) + reader := bufio.NewReader(client) + response := func() string { + var lines []string + for { + line, err := reader.ReadString('\n') + require.NoError(t, err) + if line == "\n" { + return strings.Join(lines, "") + } + lines = append(lines, line) + } + } + // Removing a peer through UAPI would leave the transport's handle stale. + _, err = fmt.Fprintf(client, "set=1\npublic_key=%s\nremove=true\n\n", hex.EncodeToString(remote[:])) + require.NoError(t, err) + require.NotContains(t, response(), "errno=0") + _, err = io.WriteString(client, "get=1\n\n") + require.NoError(t, err) + status := response() + require.Contains(t, status, "public_key="+hex.EncodeToString(remote[:])) + require.Contains(t, status, "errno=0") + require.Len(t, wg.Peers(), 1) + require.Same(t, peer, wg.Peers()[0].Peer) +} diff --git a/polyamide/transports/wireguard/conn/bind_std.go b/polyamide/transports/wireguard/conn/bind_std.go index effe124d..6f4ee827 100644 --- a/polyamide/transports/wireguard/conn/bind_std.go +++ b/polyamide/transports/wireguard/conn/bind_std.go @@ -16,7 +16,6 @@ import ( "sync" "syscall" - "github.com/encodeous/nylon/perf" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" ) @@ -80,7 +79,9 @@ type StdNetEndpoint struct { // src is the current sticky source address and interface index, if // supported. Typically this is a PKTINFO structure from/for control // messages, see unix.PKTINFO for an example. - src []byte + // Peers can share an endpoint. Source state can change while another peer sends. + srcMu sync.RWMutex + src []byte } var ( @@ -99,6 +100,8 @@ func (*StdNetBind) ParseEndpoint(s string) (Endpoint, error) { } func (e *StdNetEndpoint) ClearSrc() { + e.srcMu.Lock() + defer e.srcMu.Unlock() if e.src != nil { // Truncate src, no need to reallocate. e.src = e.src[:0] @@ -266,7 +269,6 @@ func (s *StdNetBind) receiveIP( return 0, err } } - perf.RecvBatchSize.Add(float64(numMsgs)) } else { msg := &(*msgs)[0] msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB) @@ -275,7 +277,6 @@ func (s *StdNetBind) receiveIP( } numMsgs = 1 } - perf.RecvsPerSecond.Add(1) for i := 0; i < numMsgs; i++ { msg := &(*msgs)[i] sizes[i] = msg.N @@ -436,8 +437,6 @@ func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message if runtime.GOOS == "linux" || runtime.GOOS == "android" { for { n, err = pc.WriteBatch(msgs[start:], 0) - perf.SendBatchSize.Add(float64(n)) - perf.SendsPerSecond.Add(1) if err != nil || n == len(msgs[start:]) { break } @@ -450,8 +449,6 @@ func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message break } } - perf.SendsPerSecond.Add(float64(len(msgs))) - perf.SendBatchSize.Add(1) } return err } diff --git a/polyamide/transports/wireguard/conn/sticky_linux.go b/polyamide/transports/wireguard/conn/sticky_linux.go index adfedc17..7d54224a 100644 --- a/polyamide/transports/wireguard/conn/sticky_linux.go +++ b/polyamide/transports/wireguard/conn/sticky_linux.go @@ -15,6 +15,8 @@ import ( ) func (e *StdNetEndpoint) SrcIP() netip.Addr { + e.srcMu.RLock() + defer e.srcMu.RUnlock() switch len(e.src) { case unix.CmsgSpace(unix.SizeofInet4Pktinfo): info := (*unix.Inet4Pktinfo)(unsafe.Pointer(&e.src[unix.CmsgLen(0)])) @@ -30,6 +32,8 @@ func (e *StdNetEndpoint) SrcIP() netip.Addr { } func (e *StdNetEndpoint) SrcIfidx() int32 { + e.srcMu.RLock() + defer e.srcMu.RUnlock() switch len(e.src) { case unix.CmsgSpace(unix.SizeofInet4Pktinfo): info := (*unix.Inet4Pktinfo)(unsafe.Pointer(&e.src[unix.CmsgLen(0)])) @@ -48,7 +52,9 @@ func (e *StdNetEndpoint) SrcToString() string { // getSrcFromControl parses the control for PKTINFO and if found updates ep with // the source information found. func getSrcFromControl(control []byte, ep *StdNetEndpoint) { - ep.ClearSrc() + ep.srcMu.Lock() + defer ep.srcMu.Unlock() + ep.src = ep.src[:0] var ( hdr unix.Cmsghdr @@ -98,6 +104,8 @@ func getSrcFromControl(control []byte, ep *StdNetEndpoint) { // and source ifindex found in ep. control's len will be set to 0 in the event // that ep is a default value. func setSrcControl(control *[]byte, ep *StdNetEndpoint) { + ep.srcMu.RLock() + defer ep.srcMu.RUnlock() if cap(*control) < len(ep.src) { return } diff --git a/polyamide/transports/wireguard/conn/sticky_linux_test.go b/polyamide/transports/wireguard/conn/sticky_linux_test.go index 1b1ee683..d3660790 100644 --- a/polyamide/transports/wireguard/conn/sticky_linux_test.go +++ b/polyamide/transports/wireguard/conn/sticky_linux_test.go @@ -12,6 +12,7 @@ import ( "net" "net/netip" "runtime" + "sync" "testing" "unsafe" @@ -264,3 +265,26 @@ func Test_listenConfig(t *testing.T) { } }) } + +func TestSharedEndpointSourceState(t *testing.T) { + source := new(StdNetEndpoint) + setSrc(source, netip.MustParseAddr("127.0.0.1"), 1) + control := append([]byte(nil), source.src...) + endpoint := new(StdNetEndpoint) + var workers sync.WaitGroup + for _, work := range []func(){ + func() { endpoint.ClearSrc() }, + func() { getSrcFromControl(control, endpoint) }, + func() { endpoint.SrcIP(); endpoint.SrcIfidx(); endpoint.SrcToString() }, + func() { var output []byte; setSrcControl(&output, endpoint) }, + } { + workers.Add(1) + go func() { + defer workers.Done() + for range 1000 { + work() + } + }() + } + workers.Wait() +} diff --git a/polyamide/transports/wireguard/device/device.go b/polyamide/transports/wireguard/device/device.go index 880d5112..8cebbedb 100644 --- a/polyamide/transports/wireguard/device/device.go +++ b/polyamide/transports/wireguard/device/device.go @@ -64,7 +64,8 @@ type Device struct { limiter ratelimiter.Ratelimiter } - TCFilters []TCFilter + // hooks are fixed at construction. + hooks Hooks Allowedips AllowedIPs indexTable IndexTable cookieChecker CookieChecker @@ -75,7 +76,6 @@ type Device struct { messageBuffers *WaitPool inboundElements *WaitPool outboundElements *WaitPool - tcElements *WaitPool } queue struct { @@ -286,8 +286,15 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error { return nil } +// NewDevice creates a device that routes packets between its TUN and peers by allowed IPs. func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger) *Device { + return NewDeviceWithHooks(tunDevice, bind, logger, Hooks{}) +} + +// NewDeviceWithHooks creates a device that reports packets and endpoints to hooks. +func NewDeviceWithHooks(tunDevice tun.Device, bind conn.Bind, logger *Logger, hooks Hooks) *Device { device := new(Device) + device.hooks = hooks device.state.state.Store(uint32(deviceStateDown)) device.closed = make(chan struct{}) device.Log = logger @@ -302,9 +309,6 @@ func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger) *Device { device.peers.keyMap = make(map[NoisePublicKey]*Peer) device.rate.limiter.Init() device.indexTable.Init() - device.TCFilters = append(device.TCFilters, TCFDrop) - device.TCFilters = append(device.TCFilters, TCFBounce) - device.TCFilters = append(device.TCFilters, TCFAllowedip) device.PopulatePools() diff --git a/polyamide/transports/wireguard/device/packets.go b/polyamide/transports/wireguard/device/packets.go new file mode 100644 index 00000000..55fcd817 --- /dev/null +++ b/polyamide/transports/wireguard/device/packets.go @@ -0,0 +1,92 @@ +package device + +import ( + "encoding/binary" + "net" + + "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +// InboundPacket is a decrypted packet. Packet is valid only during the Receiver call. +type InboundPacket struct { + Packet []byte + Endpoint conn.Endpoint +} + +// Hooks let the owner of a device handle its packets and endpoints. +type Hooks struct { + // Receive handles decrypted packets from peer. Without it, the device writes + // packets from allowed source addresses to its TUN, like upstream WireGuard. + Receive func(peer *Peer, packets []InboundPacket) + // EndpointLearned reports an authenticated packet from an endpoint that is not + // in the peer's endpoint list. It runs on the receive path and must not block. + EndpointLearned func(peer *Peer, endpoint conn.Endpoint) +} + +// OutboundPacket is a plaintext packet stored at MessageTransportHeaderSize in a +// buffer from GetMessageBuffer. +type OutboundPacket struct { + Buffer *[MaxMessageSize]byte + Size int + Endpoint conn.Endpoint // nil uses the peer's endpoints +} + +// SendPackets encrypts and sends packets to peer. It takes ownership of every buffer. +func (peer *Peer) SendPackets(packets []OutboundPacket) { + device := peer.device + if !peer.isRunning.Load() { + for _, packet := range packets { + device.PutMessageBuffer(packet.Buffer) + } + return + } + elems := device.GetOutboundElementsContainer() + for _, packet := range packets { + elem := device.GetOutboundElement() + elem.nonce = 0 + elem.endpoint = packet.Endpoint + elem.buffer = packet.Buffer + elem.packet = packet.Buffer[MessageTransportHeaderSize : MessageTransportHeaderSize+packet.Size] + elems.elems = append(elems.elems, elem) + } + peer.StagePackets(elems) + peer.SendStagedPackets() +} + +// allowedPacket trims packet to its IP length and checks that peer may use its source address. +func (device *Device) allowedPacket(peer *Peer, packet []byte) ([]byte, bool) { + switch packet[0] >> 4 { + case 4: + if len(packet) < ipv4.HeaderLen { + return nil, false + } + length := binary.BigEndian.Uint16(packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2]) + if int(length) > len(packet) || int(length) < ipv4.HeaderLen { + return nil, false + } + packet = packet[:length] + if device.Allowedips.Lookup(packet[IPv4offsetSrc:IPv4offsetSrc+net.IPv4len]) != peer { + device.Log.Verbosef("IPv4 packet with disallowed source address from %v", peer) + return nil, false + } + case 6: + if len(packet) < ipv6.HeaderLen { + return nil, false + } + length := binary.BigEndian.Uint16(packet[IPv6offsetPayloadLength:IPv6offsetPayloadLength+2]) + ipv6.HeaderLen + if int(length) > len(packet) { + return nil, false + } + packet = packet[:length] + if device.Allowedips.Lookup(packet[IPv6offsetSrc:IPv6offsetSrc+net.IPv6len]) != peer { + device.Log.Verbosef("IPv6 packet with disallowed source address from %v", peer) + return nil, false + } + default: + device.Log.Verbosef("Packet with invalid IP version from %v", peer) + return nil, false + } + return packet, true +} diff --git a/polyamide/transports/wireguard/device/peer.go b/polyamide/transports/wireguard/device/peer.go index 1da20f7a..c4bb4356 100644 --- a/polyamide/transports/wireguard/device/peer.go +++ b/polyamide/transports/wireguard/device/peer.go @@ -13,7 +13,6 @@ import ( "sync/atomic" "time" - "github.com/encodeous/nylon/perf" "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" ) @@ -171,8 +170,6 @@ func (peer *Peer) SendBuffers(buffers [][]byte, eps []conn.Endpoint) error { for _, b := range buffers { totalLen += uint64(len(b)) } - perf.SentPacketPerSecond.Add(float64(len(buffers))) - perf.SentBytesPerSecond.Add(float64(totalLen)) peer.txBytes.Add(totalLen) return anyError } @@ -315,11 +312,13 @@ func (peer *Peer) Stop() { func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) { peer.endpoints.Lock() - defer peer.endpoints.Unlock() if peer.endpoints.disableRoaming { + peer.endpoints.Unlock() return } peer.endpoints.clearSrcOnTx = false + destination := endpoint.DstIPPort() + learned := !slices.ContainsFunc(peer.endpoints.val, func(known conn.Endpoint) bool { return known.DstIPPort() == destination }) if peer.endpoints.preferRoaming || len(peer.endpoints.val) == 0 { if len(peer.endpoints.val) == 0 { @@ -328,6 +327,10 @@ func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) { peer.endpoints.val[0] = endpoint } } + peer.endpoints.Unlock() + if learned && peer.device.hooks.EndpointLearned != nil { + peer.device.hooks.EndpointLearned(peer, endpoint) + } } // SetEndpoints configures the endpoints of the peer. The first endpoint will be the default endpoint used for packet routing @@ -338,14 +341,14 @@ func (peer *Peer) SetEndpoints(endpoints []conn.Endpoint) { } func (peer *Peer) GetEndpoints() []conn.Endpoint { - peer.handshake.mutex.RLock() - defer peer.handshake.mutex.RUnlock() + peer.endpoints.Lock() + defer peer.endpoints.Unlock() return slices.Clone(peer.endpoints.val) } func (peer *Peer) CleanEndpoints() { - peer.handshake.mutex.Lock() - defer peer.handshake.mutex.Unlock() + peer.endpoints.Lock() + defer peer.endpoints.Unlock() if len(peer.endpoints.val) > 1 { peer.endpoints.val = peer.endpoints.val[:1] } @@ -358,8 +361,8 @@ func (peer *Peer) SetPreferRoaming(val bool) { } func (peer *Peer) GetPreferRoaming() bool { - peer.handshake.mutex.RLock() - defer peer.handshake.mutex.RUnlock() + peer.endpoints.Lock() + defer peer.endpoints.Unlock() return peer.endpoints.preferRoaming } diff --git a/polyamide/transports/wireguard/device/pools.go b/polyamide/transports/wireguard/device/pools.go index 03a50bef..5dec7618 100644 --- a/polyamide/transports/wireguard/device/pools.go +++ b/polyamide/transports/wireguard/device/pools.go @@ -64,9 +64,6 @@ func (device *Device) PopulatePools() { device.pool.outboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any { return new(QueueOutboundElement) }) - device.pool.tcElements = NewWaitPool(PreallocatedBuffersPerPool, func() any { - return new(TCElement) - }) } func (device *Device) GetInboundElementsContainer() *QueueInboundElementsContainer { @@ -108,16 +105,6 @@ func (device *Device) PutMessageBuffer(msg *[MaxMessageSize]byte) { device.pool.messageBuffers.Put(msg) } -func (device *Device) GetTCElement() *TCElement { - return device.pool.tcElements.Get().(*TCElement) -} - -func (device *Device) PutTCElement(elem *TCElement) { - elem.clearPointers() - elem.Priority = 0 - device.pool.tcElements.Put(elem) -} - func (device *Device) GetInboundElement() *QueueInboundElement { return device.pool.inboundElements.Get().(*QueueInboundElement) } diff --git a/polyamide/transports/wireguard/device/receive.go b/polyamide/transports/wireguard/device/receive.go index fcff4b6b..2dd6226a 100644 --- a/polyamide/transports/wireguard/device/receive.go +++ b/polyamide/transports/wireguard/device/receive.go @@ -12,7 +12,6 @@ import ( "sync" "time" - "github.com/encodeous/nylon/perf" "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" "golang.org/x/crypto/chacha20poly1305" ) @@ -434,8 +433,8 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { device.Log.Verbosef("%v - Routine: sequential receiver - started", peer) var ( - tcBufs = make([]*TCElement, 0, maxBatchSize) - tcs = NewTCState() + inbound = make([]InboundPacket, 0, maxBatchSize) + bufs = make([][]byte, 0, maxBatchSize) ) for elemsContainer := range peer.queue.inbound.c { @@ -446,7 +445,6 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { validTailPacket := -1 dataPacketReceived := false rxBytesLen := uint64(0) - rxPkts := 0 for i, elem := range elemsContainer.elems { if elem.packet == nil { @@ -465,7 +463,6 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { peer.SendStagedPackets() } rxBytesLen += uint64(len(elem.packet) + MinMessageSize) - rxPkts++ if len(elem.packet) == 0 { device.Log.Verbosef("%v - Receiving keepalive packet", peer) @@ -473,25 +470,14 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { } dataPacketReceived = true - tce := device.GetTCElement() - tce.Packet = elem.packet - tce.Buffer = elem.buffer - elem.buffer = nil - elem.packet = nil - tce.FromEp = elem.endpoint - tce.FromPeer = peer - - tcBufs = append(tcBufs, tce) + if device.hooks.Receive != nil { + inbound = append(inbound, InboundPacket{Packet: elem.packet, Endpoint: elem.endpoint}) + } else if packet, ok := device.allowedPacket(peer, elem.packet); ok { + bufs = append(bufs, elem.buffer[:MessageTransportOffsetContent+len(packet)]) + } } - // pass to traffic control - device.TCBatch(tcBufs, tcs) - - tcBufs = tcBufs[:0] - peer.rxBytes.Add(rxBytesLen) - perf.RecvBytesPerSecond.Add(float64(rxBytesLen)) - perf.RecvPacketPerSecond.Add(float64(rxPkts)) if validTailPacket >= 0 { peer.SetEndpointFromPacket(elemsContainer.elems[validTailPacket].endpoint) peer.keepKeyFreshReceiving() @@ -501,6 +487,19 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { if dataPacketReceived { peer.timersDataReceived() } + if len(inbound) > 0 { + device.hooks.Receive(peer, inbound) + clear(inbound) + inbound = inbound[:0] + } + if len(bufs) > 0 { + _, err := device.tun.device.Write(bufs, MessageTransportOffsetContent) + if err != nil && !device.isClosed() { + device.Log.Errorf("Failed to write packets to TUN device: %v", err) + } + clear(bufs) + bufs = bufs[:0] + } for _, elem := range elemsContainer.elems { device.PutMessageBuffer(elem.buffer) device.PutInboundElement(elem) diff --git a/polyamide/transports/wireguard/device/send.go b/polyamide/transports/wireguard/device/send.go index 2e091b67..bfaf07fc 100644 --- a/polyamide/transports/wireguard/device/send.go +++ b/polyamide/transports/wireguard/device/send.go @@ -8,6 +8,7 @@ package device import ( "encoding/binary" "errors" + "net" "os" "sync" "time" @@ -15,6 +16,8 @@ import ( "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" "github.com/encodeous/nylon/polyamide/transports/wireguard/tun" "golang.org/x/crypto/chacha20poly1305" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" ) /* Outbound flow @@ -230,15 +233,19 @@ func (device *Device) RoutineReadFromTUN() { bufs = make([]*[MaxMessageSize]byte, batchSize) count = batchSize sizes = make([]int, batchSize) - tcBufs = make([]*TCElement, 0, batchSize) offset = MessageTransportHeaderSize - tcs = NewTCState() + byPeer = make(map[*Peer][]OutboundPacket) ) for i := 0; i < batchSize; i++ { bufs[i] = device.GetMessageBuffer() rBufs[i] = bufs[i][:] } + defer func() { + for _, buf := range bufs { + device.PutMessageBuffer(buf) + } + }() for { count, readErr = device.tun.device.Read(rBufs, sizes, offset) @@ -247,19 +254,33 @@ func (device *Device) RoutineReadFromTUN() { if sizes[i] < 1 { continue } - tce := device.GetTCElement() - tce.Buffer = bufs[i] - tce.Packet = bufs[i][offset : offset+sizes[i]] - tcBufs = append(tcBufs, tce) - + // lookup peer + packet := bufs[i][offset : offset+sizes[i]] + var peer *Peer + switch packet[0] >> 4 { + case 4: + if len(packet) >= ipv4.HeaderLen { + peer = device.Allowedips.Lookup(packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len]) + } + case 6: + if len(packet) >= ipv6.HeaderLen { + peer = device.Allowedips.Lookup(packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len]) + } + default: + device.Log.Verbosef("Received packet with unknown IP version") + } + if peer == nil { + continue + } + byPeer[peer] = append(byPeer[peer], OutboundPacket{Buffer: bufs[i], Size: sizes[i]}) bufs[i] = device.GetMessageBuffer() rBufs[i] = bufs[i][:] } - // pass to traffic control - device.TCBatch(tcBufs, tcs) - - tcBufs = tcBufs[:0] + for peer, packets := range byPeer { + peer.SendPackets(packets) + delete(byPeer, peer) + } if readErr != nil { if errors.Is(readErr, tun.ErrTooManySegments) { diff --git a/polyamide/transports/wireguard/device/traffic_control.go b/polyamide/transports/wireguard/device/traffic_control.go deleted file mode 100644 index 6b23f32e..00000000 --- a/polyamide/transports/wireguard/device/traffic_control.go +++ /dev/null @@ -1,217 +0,0 @@ -package device - -import ( - "slices" - - "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" -) - -// polyamide traffic control provides a facility to re-order, manipulate, and redirect packets between nylon/polyamide nodes -// this facility operates at the IP/polysock level - -type TCAction int -type TCPriority int - -const ( - // TcPass will pass the packet on to the next layer - TcPass TCAction = iota - // TcBounce will bounce the packet back to the system for handling - TcBounce - // TcForward will send the packet through nylon/polyamide. toPeer must be set in TCElement - TcForward - // TcDrop will completely drop the packet - TcDrop -) - -const ( - TcNormalPriority TCPriority = iota - TcMediumPriority - TcHighPriority - TcMaxPriority -) - -type TCFilter func(dev *Device, packet *TCElement) (TCAction, error) - -func TCFAllowedip(dev *Device, packet *TCElement) (TCAction, error) { - if packet.ToPeer != nil { - return TcForward, nil - } - peer := dev.Allowedips.Lookup(packet.GetDstBytes()) - if peer != nil { - packet.ToPeer = peer - //fmt.Printf("fw: %s -> %s\n", packet.GetDst().String(), peer) - return TcForward, nil - } - - //fmt.Printf("nfw addr: %s\n", packet.GetDst().String()) - return TcPass, nil -} - -func TCFDrop(dev *Device, packet *TCElement) (TCAction, error) { - //dev.Log.Verbosef("TCFDrop packet: %v -> %v", packet.GetSrc(), packet.GetDst()) - return TcDrop, nil -} - -func TCFBounce(dev *Device, packet *TCElement) (TCAction, error) { - if packet.Incoming() { - //dev.Log.Verbosef("TCFBounce packet: %v -> %v", packet.GetSrc(), packet.GetDst()) - return TcBounce, nil - } - return TcPass, nil -} - -type TCElement struct { - Buffer *[MaxMessageSize]byte // slice holding the packet data - Packet []byte // slice of "buffer" (always!) - FromEp conn.Endpoint // what the source wireguard UDP endpoint (if any) is - ToEp conn.Endpoint // which wireguard UDP endpoint to send this Packet to - FromPeer *Peer // which peer (if any) sent us this Packet - ToPeer *Peer // which peer to send this Packet to - Priority TCPriority // Priority, higher is better -} - -func (elem *TCElement) clearPointers() { - elem.Buffer = nil - elem.Packet = nil - elem.FromEp = nil - elem.ToEp = nil - elem.FromPeer = nil - elem.ToPeer = nil -} - -func (device *Device) NewTCElement() *TCElement { - elem := device.GetTCElement() - elem.Buffer = device.GetMessageBuffer() - return elem -} - -func (device *Device) InstallFilter(filter TCFilter) { - device.TCFilters = append(device.TCFilters, filter) -} - -type TCState struct { - priority [][]*TCElement - bouncePkts []*TCElement - bounceBufs [][]byte - elemsForPeer map[*Peer][]*TCElement -} - -func NewTCState() *TCState { - return &TCState{ - priority: make([][]*TCElement, TcMaxPriority+1), - bouncePkts: make([]*TCElement, 0, conn.IdealBatchSize), - bounceBufs: make([][]byte, 0, conn.IdealBatchSize), - elemsForPeer: make(map[*Peer][]*TCElement), - } -} - -func (device *Device) TCBatch(batch []*TCElement, tcs *TCState) { - for i, elem := range batch { - // process TC Filters - act := TcPass - if !elem.ParsePacket() || !elem.Validate() { - device.Log.Errorf("Found malformed packet, dropping packet") - act = TcDrop - } else { - for _, filter := range slices.Backward(device.TCFilters) { - nAct, err := filter(device, elem) - act = nAct - if err != nil { - device.Log.Errorf("Error on filter action: %v", err) - act = TcDrop - } - if act != TcPass { - break - } - } - } - if act == TcPass { - device.Log.Errorf("Unexpectedly passed all filters!") - act = TcDrop - } - - batch[i] = nil - - switch act { - case TcDrop: - // cleanup - device.PutMessageBuffer(elem.Buffer) - device.PutTCElement(elem) - case TcBounce: - // bounce back to system - tcs.bouncePkts = append(tcs.bouncePkts, elem) - case TcForward: - // reroute/forward packet - if elem.ToPeer == nil { - device.Log.Errorf("Failed to forward packet to destination, toPeer not set") - device.PutMessageBuffer(elem.Buffer) - device.PutTCElement(elem) - continue - } - tcs.priority[elem.Priority] = append(tcs.priority[elem.Priority], elem) - default: - panic("unreachable default case") - } - } - - // bounce packets back to the system - if len(tcs.bouncePkts) > 0 { - for _, elem := range tcs.bouncePkts { - tcs.bounceBufs = append(tcs.bounceBufs, elem.Buffer[:MessageTransportHeaderSize+len(elem.Packet)]) - } - // here, we need to use elem.Buffer instead of elem.Packet since we will get io.ErrShortBuffer if offset < 4 - _, err := device.tun.device.Write(tcs.bounceBufs, MessageTransportHeaderSize) - if err != nil && !device.isClosed() { - device.Log.Errorf("Failed to loop back packets to TUN device: %v", err) - } - for i, elem := range tcs.bouncePkts { - device.PutMessageBuffer(elem.Buffer) - device.PutTCElement(elem) - tcs.bouncePkts[i] = nil - tcs.bounceBufs[i] = nil - } - tcs.bounceBufs = tcs.bounceBufs[:0] - tcs.bouncePkts = tcs.bouncePkts[:0] - } - - // forward packets to peers based on priority - for p, elems := range slices.Backward(tcs.priority) { - for i, elem := range elems { - if elem == nil { - continue - } - tcs.elemsForPeer[elem.ToPeer] = append(tcs.elemsForPeer[elem.ToPeer], elem) - elems[i] = nil - } - tcs.priority[p] = tcs.priority[p][:0] - } - - // stage packets to peers - for peer, elems := range tcs.elemsForPeer { - if len(elems) == 0 { - continue - } - if peer.isRunning.Load() { - obec := device.GetOutboundElementsContainer() - for i, elem := range elems { - obe := device.GetOutboundElement() - obe.nonce = 0 - obe.endpoint = elem.ToEp - obe.packet = elem.Packet - obe.buffer = elem.Buffer - obec.elems = append(obec.elems, obe) - device.PutTCElement(elem) - elems[i] = nil - } - peer.StagePackets(obec) - peer.SendStagedPackets() - } else { - for i, elem := range elems { - device.PutMessageBuffer(elem.Buffer) - device.PutTCElement(elem) - elems[i] = nil - } - } - tcs.elemsForPeer[peer] = elems[:0] - } -} diff --git a/polyamide/transports/wireguard/device/traffic_manip.go b/polyamide/transports/wireguard/device/traffic_manip.go deleted file mode 100644 index 89d956eb..00000000 --- a/polyamide/transports/wireguard/device/traffic_manip.go +++ /dev/null @@ -1,214 +0,0 @@ -package device - -import ( - "encoding/binary" - "golang.org/x/net/ipv4" - "golang.org/x/net/ipv6" - "net" - "net/netip" -) - -// poly packets use other "IP Versions" -const ( - PolyHeaderSize = 3 - PolyOffsetPayloadLength = 1 -) - -func (elem *TCElement) InitPacket(ver int, len uint16) { - elem.Packet = elem.Buffer[MessageTransportHeaderSize : MessageTransportHeaderSize+len] - elem.SetIPVersion(ver) - elem.SetLength(len) -} - -func (elem *TCElement) ParsePacket() bool { - if elem == nil { - return false - } - if len(elem.Packet) == 0 { - if elem.Buffer == nil { - return false - } - elem.Packet = elem.Buffer[MessageTransportHeaderSize:] - } - if !elem.hasPacketHeader() { - return false - } - l := elem.GetLength() - if int(l) > len(elem.Packet) { - return false - } - elem.Packet = elem.Packet[:l] - return true -} - -func (elem *TCElement) Incoming() bool { - return elem.FromPeer != nil -} - -func (elem *TCElement) GetIPVersion() int { - return int(elem.Packet[0] >> 4) -} - -func (elem *TCElement) GetSrcBytes() []byte { - ver := elem.GetIPVersion() - if ver == 4 { - return elem.Packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len] - } else if ver == 6 { - return elem.Packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len] - } - return nil -} - -func (elem *TCElement) GetDstBytes() []byte { - ver := elem.GetIPVersion() - if ver == 4 { - return elem.Packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len] - } else if ver == 6 { - return elem.Packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len] - } - return nil -} - -func (elem *TCElement) GetSrc() netip.Addr { - ver := elem.GetIPVersion() - b := elem.GetSrcBytes() - if ver == 4 { - return netip.AddrFrom4([4]byte(b)) - } else if ver == 6 { - return netip.AddrFrom16([16]byte(b)) - } - return netip.IPv4Unspecified() -} - -func (elem *TCElement) SetSrc(addr netip.Addr) { - bin, err := addr.MarshalBinary() - if err != nil { - panic(err) - } - copy(elem.GetSrcBytes(), bin) -} - -func (elem *TCElement) GetDst() netip.Addr { - ver := elem.GetIPVersion() - b := elem.GetDstBytes() - if ver == 4 { - return netip.AddrFrom4([4]byte(b)) - } else if ver == 6 { - return netip.AddrFrom16([16]byte(b)) - } - return netip.IPv4Unspecified() -} - -func (elem *TCElement) SetDst(addr netip.Addr) { - bin, err := addr.MarshalBinary() - if err != nil { - panic(err) - } - copy(elem.GetDstBytes(), bin) -} - -func (elem *TCElement) SetIPVersion(ver int) { - elem.Packet[0] = byte(ver << 4) -} - -// GetLength returns the length of the packet, including the header -func (elem *TCElement) GetLength() uint16 { - ver := elem.GetIPVersion() - if ver == 4 { - field := elem.Packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2] - return binary.BigEndian.Uint16(field) - } else if ver == 6 { - field := elem.Packet[IPv6offsetPayloadLength : IPv6offsetPayloadLength+2] - return binary.BigEndian.Uint16(field) + ipv6.HeaderLen - } else { - field := elem.Packet[PolyOffsetPayloadLength : PolyOffsetPayloadLength+2] - return binary.BigEndian.Uint16(field) + PolyHeaderSize - } -} - -func (elem *TCElement) SetLength(len uint16) { - ver := elem.GetIPVersion() - if ver == 4 { - binary.BigEndian.PutUint16(elem.Packet[IPv4offsetTotalLength:IPv4offsetTotalLength+2], len) - } else if ver == 6 { - binary.BigEndian.PutUint16(elem.Packet[IPv6offsetPayloadLength:IPv6offsetPayloadLength+2], len-ipv6.HeaderLen) - } else { - binary.BigEndian.PutUint16(elem.Packet[PolyOffsetPayloadLength:PolyOffsetPayloadLength+2], len-PolyHeaderSize) - } -} - -func (elem *TCElement) Payload() []byte { - ver := elem.GetIPVersion() - if ver == 4 { - return elem.Packet[ipv4.HeaderLen:] - } else if ver == 6 { - return elem.Packet[ipv6.HeaderLen:] - } else { - return elem.Packet[PolyHeaderSize:] - } -} - -func (elem *TCElement) Validate() bool { - if elem == nil || len(elem.Packet) == 0 { - return false - } - if !elem.hasPacketHeader() { - return false - } - ver := elem.GetIPVersion() - if ver == 4 { - if elem.GetLength() < ipv4.HeaderLen { - return false - } - } else if ver == 6 { - if elem.GetLength() < ipv6.HeaderLen { - return false - } - } else { - if elem.GetLength() < PolyHeaderSize { - return false - } - } - return true -} - -func (elem *TCElement) hasPacketHeader() bool { - if elem == nil || len(elem.Packet) < PolyHeaderSize { - return false - } - ver := elem.GetIPVersion() - if ver == 4 { - return len(elem.Packet) >= ipv4.HeaderLen - } else if ver == 6 { - return len(elem.Packet) >= ipv6.HeaderLen - } - return true -} - -func (elem *TCElement) TTLBytes() []byte { - if elem.GetIPVersion() == 4 { - return elem.Packet[8:9] - } else if elem.GetIPVersion() == 6 { - return elem.Packet[7:8] - } else { - panic("invalid IP version") - } -} - -func (elem *TCElement) GetTTL() byte { - return elem.TTLBytes()[0] -} - -func (elem *TCElement) DecrementTTL() { - const ( - checksumOffset = 10 - ) - elem.TTLBytes()[0]-- - // using algorithm described in https://datatracker.ietf.org/doc/html/rfc1141 - if elem.GetIPVersion() == 4 { - // fast incremental checksum calculation - csum := uint32(binary.BigEndian.Uint16(elem.Packet[checksumOffset:checksumOffset+2])) + 0x100 - csum = csum + (csum >> 16) - binary.BigEndian.PutUint16(elem.Packet[checksumOffset:checksumOffset+2], uint16(csum)) - } -} diff --git a/polyamide/transports/wireguard/device/traffic_manip_test.go b/polyamide/transports/wireguard/device/traffic_manip_test.go deleted file mode 100644 index bf9e3ad6..00000000 --- a/polyamide/transports/wireguard/device/traffic_manip_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package device - -import ( - "encoding/binary" - "testing" -) - -func TestParsePacketRejectsLengthBeyondReadBuffer(t *testing.T) { - var buf [MaxMessageSize]byte - packet := buf[MessageTransportHeaderSize : MessageTransportHeaderSize+2016] - packet[0] = 6 << 4 - binary.BigEndian.PutUint16(packet[IPv6offsetPayloadLength:IPv6offsetPayloadLength+2], 2010) - - elem := &TCElement{ - Buffer: &buf, - Packet: packet, - } - - if elem.ParsePacket() { - t.Fatal("expected oversized packet to be rejected") - } -} - -func TestParsePacketTrimsToAdvertisedLength(t *testing.T) { - var buf [MaxMessageSize]byte - packet := buf[MessageTransportHeaderSize : MessageTransportHeaderSize+128] - packet[0] = 6 << 4 - binary.BigEndian.PutUint16(packet[IPv6offsetPayloadLength:IPv6offsetPayloadLength+2], 8) - - elem := &TCElement{ - Buffer: &buf, - Packet: packet, - } - - if !elem.ParsePacket() { - t.Fatal("expected valid packet to parse") - } - if len(elem.Packet) != 48 { - t.Fatalf("expected packet length 48, got %d", len(elem.Packet)) - } -} diff --git a/polyamide/transports/wireguard/device/uapi.go b/polyamide/transports/wireguard/device/uapi.go index 4bae86b4..a1315a12 100644 --- a/polyamide/transports/wireguard/device/uapi.go +++ b/polyamide/transports/wireguard/device/uapi.go @@ -420,6 +420,15 @@ func (device *Device) IpcSet(uapiConf string) error { } func (device *Device) IpcHandle(socket net.Conn) { + device.ipcHandle(socket, true) +} + +// IpcHandleReadOnly serves UAPI like IpcHandle, but rejects set operations. +func (device *Device) IpcHandleReadOnly(socket net.Conn) { + device.ipcHandle(socket, false) +} + +func (device *Device) ipcHandle(socket net.Conn, allowSet bool) { defer socket.Close() buffered := func(s io.ReadWriter) *bufio.ReadWriter { @@ -437,7 +446,17 @@ func (device *Device) IpcHandle(socket net.Conn) { // handle operation switch op { case "set=1\n": - err = device.IpcSetOperation(buffered.Reader) + if allowSet { + err = device.IpcSetOperation(buffered.Reader) + break + } + // Discard the request, which ends with an empty line, then report the error. + for line := ""; line != "\n"; { + if line, err = buffered.ReadString('\n'); err != nil { + return + } + } + err = ipcErrorf(ipc.IpcErrorInvalid, "UAPI set is not supported") case "get=1\n": var nextByte byte nextByte, err = buffered.ReadByte() diff --git a/state/crypto.go b/state/crypto.go index 886a7773..d49c8ad8 100644 --- a/state/crypto.go +++ b/state/crypto.go @@ -2,15 +2,18 @@ package state import ( "crypto/rand" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" + + "github.com/encodeous/nylon/polyamide" "go.step.sm/crypto/x25519" ) -type NyPrivateKey [device.NoisePrivateKeySize]byte -type NyPublicKey [device.NoisePublicKeySize]byte +const KeySize = polyamide.KeySize + +type NyPrivateKey [KeySize]byte +type NyPublicKey [KeySize]byte func GenerateKey() NyPrivateKey { - key := make([]byte, device.NoisePrivateKeySize) + key := make([]byte, KeySize) if _, err := rand.Read(key); err != nil { panic(err) diff --git a/state/endpoint.go b/state/endpoint.go index efef2c30..8ef19ba1 100644 --- a/state/endpoint.go +++ b/state/endpoint.go @@ -1,14 +1,10 @@ package state import ( - "fmt" "math" "slices" "sync" "time" - - "github.com/encodeous/nylon/polyamide/transports/wireguard/conn" - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" ) type Endpoint interface { @@ -29,7 +25,6 @@ type NylonEndpoint struct { lastHeardBack time.Time expRTT float64 remoteInit bool - WgEndpoint conn.Endpoint Address string } @@ -37,22 +32,6 @@ func (ep *NylonEndpoint) AsNylonEndpoint() *NylonEndpoint { return ep } -func (ep *NylonEndpoint) GetWgEndpoint(device *device.Device, resolver *EndpointResolver) (conn.Endpoint, error) { - ap, err := resolver.Get(ep.Address) - if err != nil { - return nil, err - } - - if ep.WgEndpoint == nil || ep.WgEndpoint.DstIPPort() != ap { - wgEp, err := device.Bind().ParseEndpoint(ap.String()) - if err != nil { - return nil, fmt.Errorf("failed to parse endpoint: %s, %v", ap.String(), err) - } - ep.WgEndpoint = wgEp - } - return ep.WgEndpoint, nil -} - func (n *Neighbour) BestEndpoint() Endpoint { var best Endpoint @@ -92,11 +71,10 @@ func (u *NylonEndpoint) IsAlive() bool { return u.IsActive() || !u.remoteInit // we never gc endpoints that we have in our config } -func NewEndpoint(address string, remoteInit bool, wgEndpoint conn.Endpoint, t *RouterTunables) *NylonEndpoint { +func NewEndpoint(address string, remoteInit bool, t *RouterTunables) *NylonEndpoint { return &NylonEndpoint{ t: t, remoteInit: remoteInit, - WgEndpoint: wgEndpoint, Address: address, history: make([]time.Duration, 0), expRTT: math.Inf(1), diff --git a/state/endpoint_test.go b/state/endpoint_test.go index 55d33923..dbaece52 100644 --- a/state/endpoint_test.go +++ b/state/endpoint_test.go @@ -53,7 +53,7 @@ type DataSource struct { func runTests(t *testing.T, ping func(i int) float64, dura time.Duration, fn string) (DataSource, DataSource) { t.Helper() tunables := DefaultRouterTunables() - dep := NewEndpoint("127.0.0.1:0", false, nil, &tunables) + dep := NewEndpoint("127.0.0.1:0", false, &tunables) truth := DataSource{ Name: "Truth", diff --git a/state/serialize.go b/state/serialize.go index ff963d5a..25253c25 100644 --- a/state/serialize.go +++ b/state/serialize.go @@ -3,8 +3,6 @@ package state import ( "encoding/base64" "fmt" - - "github.com/encodeous/nylon/polyamide/transports/wireguard/device" ) func (k NyPrivateKey) MarshalText() ([]byte, error) { @@ -18,8 +16,8 @@ func (k *NyPrivateKey) UnmarshalText(text []byte) error { if err != nil { return fmt.Errorf("failed to decode private key: %w", err) } - if len(data) != device.NoisePrivateKeySize { - return fmt.Errorf("private key must decode to %d bytes, got %d", device.NoisePrivateKeySize, len(data)) + if len(data) != KeySize { + return fmt.Errorf("private key must decode to %d bytes, got %d", KeySize, len(data)) } *k = NyPrivateKey(data) return nil @@ -29,8 +27,8 @@ func (k *NyPublicKey) UnmarshalText(text []byte) error { if err != nil { return fmt.Errorf("failed to decode public key (%s): %w", text, err) } - if len(data) != device.NoisePublicKeySize { - return fmt.Errorf("public key must decode to %d bytes, got %d", device.NoisePublicKeySize, len(data)) + if len(data) != KeySize { + return fmt.Errorf("public key must decode to %d bytes, got %d", KeySize, len(data)) } *k = NyPublicKey(data) return nil